diff --git a/apps/api/src/routes/models.ts b/apps/api/src/routes/models.ts index 3efe6c47..7b3f3a25 100644 --- a/apps/api/src/routes/models.ts +++ b/apps/api/src/routes/models.ts @@ -10,6 +10,7 @@ import { toPublicModelRecord, type ModelCredentials, } from "@kiwi/ai/models"; +import { probeModelConfiguration } from "@kiwi/ai/probe"; import { db } from "@kiwi/db"; import { AI_MODEL_ADAPTER_VALUES, @@ -61,6 +62,20 @@ const patchCredentialsSchema = z.object({ resourceName: z.string().trim().optional(), }); +// The test probe mirrors the create shape, except the API key may be omitted +// when model_id points at a stored model whose key should be reused. +const testModelSchema = z.object({ + model_id: z.string().trim().min(1).optional(), + type: modelTypeSchema, + adapter: modelAdapterSchema, + provider_model: z.string().trim().min(1), + credentials: z.object({ + apiKey: z.string().trim().min(1).optional(), + url: z.string().trim().min(1).optional(), + resourceName: z.string().trim().min(1).optional(), + }), +}); + const patchModelSchema = z.object({ display_name: z.string().trim().min(1).optional(), adapter: modelAdapterSchema.optional(), @@ -74,7 +89,11 @@ function mapModelError(status: RouteStatus, error: unknown) { return status(500, errorResponse("Internal server error", API_ERROR_CODES.INTERNAL_SERVER_ERROR)); } - switch (error.message) { + // Result.tryPromise wraps thrown errors in an UnhandledException whose + // cause carries the original error code (same pattern as mapChatError). + const errorCode = error.cause instanceof Error ? error.cause.message : error.message; + + switch (errorCode) { case API_ERROR_CODES.UNAUTHORIZED: return status(401, errorResponse("Unauthorized", API_ERROR_CODES.UNAUTHORIZED)); case API_ERROR_CODES.FORBIDDEN: @@ -266,6 +285,55 @@ export const modelsRoute = new Elysia({ prefix: "/models" }) body: createModelSchema, } ) + .post( + "/test", + async ({ body, status, user }) => + runModelAction({ + user, + status, + action: async (currentUser) => { + const membership = await requireOrganizationAdmin(currentUser); + let apiKey = body.credentials.apiKey; + + // Reusing the stored key with caller-supplied connection + // config grants nothing a PATCH does not already allow + // (kept key + new url); org admins are trusted with the + // credentials of the models they administer. + if (!apiKey) { + if (!body.model_id) { + throw new Error(API_ERROR_CODES.INVALID_MODEL); + } + const storedModel = await getModelForUpdate(db, membership.organizationId, body.model_id); + apiKey = decryptModelCredentials(storedModel.encryptedCredentials, env.AUTH_SECRET).apiKey; + } + + const credentials = normalizeCredentials({ + apiKey, + url: body.credentials.url, + resourceName: body.credentials.resourceName, + }); + const providerModel = body.provider_model.trim(); + + assertCreateModelInput({ + type: body.type, + adapter: body.adapter, + providerModel, + credentials, + }); + + return probeModelConfiguration({ + type: body.type, + adapter: body.adapter, + providerModel, + credentials, + }); + }, + success: (value) => status(200, successResponse(value)), + }), + { + body: testModelSchema, + } + ) .patch( "/:modelId", async ({ body, params, status, user }) => diff --git a/apps/frontend/components/admin/ModelFormDialog.test.tsx b/apps/frontend/components/admin/ModelFormDialog.test.tsx index e1883ad9..e898153d 100644 --- a/apps/frontend/components/admin/ModelFormDialog.test.tsx +++ b/apps/frontend/components/admin/ModelFormDialog.test.tsx @@ -2,8 +2,9 @@ import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, test, vi } from "vitest"; -const { createModel, updateModel } = vi.hoisted(() => ({ +const { createModel, testModelConnection, updateModel } = vi.hoisted(() => ({ createModel: vi.fn(), + testModelConnection: vi.fn(), updateModel: vi.fn(), })); @@ -27,6 +28,7 @@ Object.defineProperties(HTMLElement.prototype, { vi.mock("@/lib/api/models", () => ({ createModel, + testModelConnection, updateModel, })); @@ -88,6 +90,7 @@ describe("ModelFormDialog", () => { vi.clearAllMocks(); createModel.mockResolvedValue(adminModel); updateModel.mockResolvedValue(adminModel); + testModelConnection.mockResolvedValue({ ok: true }); }); test("creates a model with a model id slugged from the display name", async () => { @@ -287,6 +290,63 @@ describe("ModelFormDialog", () => { }); }); + test("blocks the save and shows the reason when the connection probe fails", async () => { + testModelConnection.mockResolvedValue({ ok: false, reason: "auth", message: "invalid api key" }); + const user = userEvent.setup(); + const onSaved = vi.fn(); + renderWithProviders(); + + await user.type(screen.getByLabelText("Anzeigename"), "GPT 4.1 Mini"); + await user.type(screen.getByLabelText("Provider-Modell-Name"), "gpt-4.1-mini"); + await user.type(screen.getByLabelText("API-Schlüssel"), "wrong-key"); + await user.click(screen.getByRole("button", { name: "Modell hinzufügen" })); + + expect( + await screen.findByText("Authentifizierung fehlgeschlagen. Prüfe den API-Schlüssel.") + ).toBeInTheDocument(); + expect(screen.getByText("invalid api key")).toBeInTheDocument(); + expect(createModel).not.toHaveBeenCalled(); + expect(onSaved).not.toHaveBeenCalled(); + }); + + test("skips the connection probe when only the display name changes", async () => { + const user = userEvent.setup(); + const onSaved = vi.fn(); + renderWithProviders( + + ); + + const nameInput = screen.getByLabelText("Anzeigename"); + await user.clear(nameInput); + await user.type(nameInput, "GPT-4.1 mini (renamed)"); + await user.click(screen.getByRole("button", { name: "Änderungen speichern" })); + + await waitFor(() => expect(onSaved).toHaveBeenCalled()); + expect(testModelConnection).not.toHaveBeenCalled(); + }); + + test("probes with the stored key reference when the key stays empty", async () => { + const user = userEvent.setup(); + const onSaved = vi.fn(); + renderWithProviders( + + ); + + const urlInput = screen.getByLabelText("Endpunkt-URL"); + await user.clear(urlInput); + await user.type(urlInput, "https://other.example.com/v1"); + await user.click(screen.getByRole("button", { name: "Änderungen speichern" })); + + await waitFor(() => expect(onSaved).toHaveBeenCalled()); + expect(testModelConnection).toHaveBeenCalledWith(expect.anything(), { + type: "text", + adapter: "openaiAPI", + provider_model: "gpt-oss-120b", + credentials: { url: "https://other.example.com/v1" }, + model_id: "local-llm", + }); + }); + test("closes without a request when nothing changed", async () => { const user = userEvent.setup(); const onOpenChange = vi.fn(); diff --git a/apps/frontend/components/admin/ModelFormDialog.tsx b/apps/frontend/components/admin/ModelFormDialog.tsx index 46cc43b2..1ea0034c 100644 --- a/apps/frontend/components/admin/ModelFormDialog.tsx +++ b/apps/frontend/components/admin/ModelFormDialog.tsx @@ -14,7 +14,7 @@ import { Label } from "@/components/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; import { ApiError } from "@/lib/api/client"; -import { createModel, updateModel } from "@/lib/api/models"; +import { createModel, testModelConnection, updateModel } from "@/lib/api/models"; import { useAppTranslations } from "@/lib/i18n/use-app-translations"; import { useApiClient } from "@/providers/ApiClientProvider"; import type { @@ -26,7 +26,7 @@ import type { ModelPatchInput, } from "@kiwi/contracts"; import { AI_MODEL_ADAPTER_VALUES } from "@kiwi/contracts"; -import { Loader2 } from "lucide-react"; +import { AlertCircle, Loader2 } from "lucide-react"; import type React from "react"; import { useEffect, useState } from "react"; import { toast } from "sonner"; @@ -79,6 +79,8 @@ export function slugifyModelId(value: string): string { .replace(/^-+|-+$/g, ""); } +type ConnectionError = { text: string; detail?: string }; + type ModelFormDialogProps = { open: boolean; onOpenChange: (open: boolean) => void; @@ -106,6 +108,7 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo const [resourceName, setResourceName] = useState(""); const [isDefault, setIsDefault] = useState(false); const [loading, setLoading] = useState(false); + const [connectionError, setConnectionError] = useState(null); useEffect(() => { if (!open) { @@ -118,12 +121,19 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo setProviderModel(model?.provider_model ?? ""); setContextWindow(String(model?.context_window ?? DEFAULT_CONTEXT_WINDOW_TOKENS)); setContextWindowError(null); + setConnectionError(null); setApiKey(""); setUrl(model?.url ?? ""); setResourceName(model?.resource_name ?? ""); setIsDefault(false); }, [open, model, type]); + // The error only describes the configuration it was tested against, so + // any change to the connection fields invalidates it. + useEffect(() => { + setConnectionError(null); + }, [adapter, providerModel, apiKey, url, resourceName]); + const adapterOptions = adapterOptionsForType(type); const modelId = modelIdTouched ? modelIdInput : slugifyModelId(displayName); const showContextWindow = supportsContextWindow(type); @@ -155,6 +165,42 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo (!requireUrl || url.trim().length > 0) && (!requireResourceName || resourceName.trim().length > 0); + // On edit a save only needs a probe when connection config changed; a + // pure rename must not fail because the provider is briefly unreachable. + const connectionChanged = + !isEdit || + !model || + adapter !== model.adapter || + providerModel.trim() !== model.provider_model || + apiKey.trim().length > 0 || + url.trim() !== (model.url ?? "") || + resourceName.trim() !== (model.resource_name ?? ""); + + // Probes the connection with the current form values; on edit a blank API + // key means the stored key is used, referenced via model_id. + const verifyConnection = async (): Promise => { + const result = await testModelConnection(apiClient, { + type, + adapter, + provider_model: providerModel.trim(), + credentials: { + ...(apiKey.trim() ? { apiKey: apiKey.trim() } : {}), + ...(url.trim() ? { url: url.trim() } : {}), + ...(resourceName.trim() ? { resourceName: resourceName.trim() } : {}), + }, + ...(isEdit && model && !apiKey.trim() ? { model_id: model.model_id } : {}), + }); + + if (!result.ok) { + setConnectionError({ + text: t(`settings.models.test.error.${result.reason}`), + detail: result.message, + }); + } + + return result.ok; + }; + const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); if (showContextWindow && contextWindowValue === null) { @@ -167,8 +213,12 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo return; } setContextWindowError(null); + setConnectionError(null); setLoading(true); try { + if (connectionChanged && !(await verifyConnection())) { + return; + } if (isEdit && model) { const patch: ModelPatchInput = {}; if (displayName.trim() !== model.display_name) { @@ -375,6 +425,17 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo ) : null} + {connectionError ? ( +
+

+ + {connectionError.text} +

+ {connectionError.detail ? ( +

{connectionError.detail}

+ ) : null} +
+ ) : null}