feat(models): verify provider connection when saving an AI model

apps/api/src/routes/models.ts

@@ -10,6 +10,7 @@ import {
     toPublicModelRecord,
     type ModelCredentials,
 } from "@kiwi/ai/models";
+import { probeModelConfiguration } from "@kiwi/ai/probe";
 import { db } from "@kiwi/db";
 import {
     AI_MODEL_ADAPTER_VALUES,
@@ -61,6 +62,20 @@ const patchCredentialsSchema = z.object({
     resourceName: z.string().trim().optional(),
 });
 
+// The test probe mirrors the create shape, except the API key may be omitted
+// when model_id points at a stored model whose key should be reused.
+const testModelSchema = z.object({
+    model_id: z.string().trim().min(1).optional(),
+    type: modelTypeSchema,
+    adapter: modelAdapterSchema,
+    provider_model: z.string().trim().min(1),
+    credentials: z.object({
+        apiKey: z.string().trim().min(1).optional(),
+        url: z.string().trim().min(1).optional(),
+        resourceName: z.string().trim().min(1).optional(),
+    }),
+});
+
 const patchModelSchema = z.object({
     display_name: z.string().trim().min(1).optional(),
     adapter: modelAdapterSchema.optional(),
@@ -74,7 +89,11 @@ function mapModelError(status: RouteStatus, error: unknown) {
         return status(500, errorResponse("Internal server error", API_ERROR_CODES.INTERNAL_SERVER_ERROR));
     }
 
-    switch (error.message) {
+    // Result.tryPromise wraps thrown errors in an UnhandledException whose
+    // cause carries the original error code (same pattern as mapChatError).
+    const errorCode = error.cause instanceof Error ? error.cause.message : error.message;
+
+    switch (errorCode) {
         case API_ERROR_CODES.UNAUTHORIZED:
             return status(401, errorResponse("Unauthorized", API_ERROR_CODES.UNAUTHORIZED));
         case API_ERROR_CODES.FORBIDDEN:
@@ -266,6 +285,55 @@ export const modelsRoute = new Elysia({ prefix: "/models" })
             body: createModelSchema,
         }
     )
+    .post(
+        "/test",
+        async ({ body, status, user }) =>
+            runModelAction({
+                user,
+                status,
+                action: async (currentUser) => {
+                    const membership = await requireOrganizationAdmin(currentUser);
+                    let apiKey = body.credentials.apiKey;
+
+                    // Reusing the stored key with caller-supplied connection
+                    // config grants nothing a PATCH does not already allow
+                    // (kept key + new url); org admins are trusted with the
+                    // credentials of the models they administer.
+                    if (!apiKey) {
+                        if (!body.model_id) {
+                            throw new Error(API_ERROR_CODES.INVALID_MODEL);
+                        }
+                        const storedModel = await getModelForUpdate(db, membership.organizationId, body.model_id);
+                        apiKey = decryptModelCredentials(storedModel.encryptedCredentials, env.AUTH_SECRET).apiKey;
+                    }
+
+                    const credentials = normalizeCredentials({
+                        apiKey,
+                        url: body.credentials.url,
+                        resourceName: body.credentials.resourceName,
+                    });
+                    const providerModel = body.provider_model.trim();
+
+                    assertCreateModelInput({
+                        type: body.type,
+                        adapter: body.adapter,
+                        providerModel,
+                        credentials,
+                    });
+
+                    return probeModelConfiguration({
+                        type: body.type,
+                        adapter: body.adapter,
+                        providerModel,
+                        credentials,
+                    });
+                },
+                success: (value) => status(200, successResponse(value)),
+            }),
+        {
+            body: testModelSchema,
+        }
+    )
     .patch(
         "/:modelId",
         async ({ body, params, status, user }) =>

apps/frontend/components/admin/ModelFormDialog.test.tsx

@@ -2,8 +2,9 @@ import { screen, waitFor } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { beforeEach, describe, expect, test, vi } from "vitest";
 
-const { createModel, updateModel } = vi.hoisted(() => ({
+const { createModel, testModelConnection, updateModel } = vi.hoisted(() => ({
     createModel: vi.fn(),
+    testModelConnection: vi.fn(),
     updateModel: vi.fn(),
 }));
 
@@ -27,6 +28,7 @@ Object.defineProperties(HTMLElement.prototype, {
 
 vi.mock("@/lib/api/models", () => ({
     createModel,
+    testModelConnection,
     updateModel,
 }));
 
@@ -88,6 +90,7 @@ describe("ModelFormDialog", () => {
         vi.clearAllMocks();
         createModel.mockResolvedValue(adminModel);
         updateModel.mockResolvedValue(adminModel);
+        testModelConnection.mockResolvedValue({ ok: true });
     });
 
     test("creates a model with a model id slugged from the display name", async () => {
@@ -287,6 +290,63 @@ describe("ModelFormDialog", () => {
         });
     });
 
+    test("blocks the save and shows the reason when the connection probe fails", async () => {
+        testModelConnection.mockResolvedValue({ ok: false, reason: "auth", message: "invalid api key" });
+        const user = userEvent.setup();
+        const onSaved = vi.fn();
+        renderWithProviders(<ModelFormDialog open onOpenChange={vi.fn()} type="text" onSaved={onSaved} />);
+
+        await user.type(screen.getByLabelText("Anzeigename"), "GPT 4.1 Mini");
+        await user.type(screen.getByLabelText("Provider-Modell-Name"), "gpt-4.1-mini");
+        await user.type(screen.getByLabelText("API-Schlüssel"), "wrong-key");
+        await user.click(screen.getByRole("button", { name: "Modell hinzufügen" }));
+
+        expect(
+            await screen.findByText("Authentifizierung fehlgeschlagen. Prüfe den API-Schlüssel.")
+        ).toBeInTheDocument();
+        expect(screen.getByText("invalid api key")).toBeInTheDocument();
+        expect(createModel).not.toHaveBeenCalled();
+        expect(onSaved).not.toHaveBeenCalled();
+    });
+
+    test("skips the connection probe when only the display name changes", async () => {
+        const user = userEvent.setup();
+        const onSaved = vi.fn();
+        renderWithProviders(
+            <ModelFormDialog open onOpenChange={vi.fn()} type="text" model={adminModel} onSaved={onSaved} />
+        );
+
+        const nameInput = screen.getByLabelText("Anzeigename");
+        await user.clear(nameInput);
+        await user.type(nameInput, "GPT-4.1 mini (renamed)");
+        await user.click(screen.getByRole("button", { name: "Änderungen speichern" }));
+
+        await waitFor(() => expect(onSaved).toHaveBeenCalled());
+        expect(testModelConnection).not.toHaveBeenCalled();
+    });
+
+    test("probes with the stored key reference when the key stays empty", async () => {
+        const user = userEvent.setup();
+        const onSaved = vi.fn();
+        renderWithProviders(
+            <ModelFormDialog open onOpenChange={vi.fn()} type="text" model={openaiApiModel} onSaved={onSaved} />
+        );
+
+        const urlInput = screen.getByLabelText("Endpunkt-URL");
+        await user.clear(urlInput);
+        await user.type(urlInput, "https://other.example.com/v1");
+        await user.click(screen.getByRole("button", { name: "Änderungen speichern" }));
+
+        await waitFor(() => expect(onSaved).toHaveBeenCalled());
+        expect(testModelConnection).toHaveBeenCalledWith(expect.anything(), {
+            type: "text",
+            adapter: "openaiAPI",
+            provider_model: "gpt-oss-120b",
+            credentials: { url: "https://other.example.com/v1" },
+            model_id: "local-llm",
+        });
+    });
+
     test("closes without a request when nothing changed", async () => {
         const user = userEvent.setup();
         const onOpenChange = vi.fn();

apps/frontend/components/admin/ModelFormDialog.tsx

@@ -14,7 +14,7 @@ import { Label } from "@/components/ui/label";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Switch } from "@/components/ui/switch";
 import { ApiError } from "@/lib/api/client";
-import { createModel, updateModel } from "@/lib/api/models";
+import { createModel, testModelConnection, updateModel } from "@/lib/api/models";
 import { useAppTranslations } from "@/lib/i18n/use-app-translations";
 import { useApiClient } from "@/providers/ApiClientProvider";
 import type {
@@ -26,7 +26,7 @@ import type {
     ModelPatchInput,
 } from "@kiwi/contracts";
 import { AI_MODEL_ADAPTER_VALUES } from "@kiwi/contracts";
-import { Loader2 } from "lucide-react";
+import { AlertCircle, Loader2 } from "lucide-react";
 import type React from "react";
 import { useEffect, useState } from "react";
 import { toast } from "sonner";
@@ -79,6 +79,8 @@ export function slugifyModelId(value: string): string {
         .replace(/^-+|-+$/g, "");
 }
 
+type ConnectionError = { text: string; detail?: string };
+
 type ModelFormDialogProps = {
     open: boolean;
     onOpenChange: (open: boolean) => void;
@@ -106,6 +108,7 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo
     const [resourceName, setResourceName] = useState("");
     const [isDefault, setIsDefault] = useState(false);
     const [loading, setLoading] = useState(false);
+    const [connectionError, setConnectionError] = useState<ConnectionError | null>(null);
 
     useEffect(() => {
         if (!open) {
@@ -118,12 +121,19 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo
         setProviderModel(model?.provider_model ?? "");
         setContextWindow(String(model?.context_window ?? DEFAULT_CONTEXT_WINDOW_TOKENS));
         setContextWindowError(null);
+        setConnectionError(null);
         setApiKey("");
         setUrl(model?.url ?? "");
         setResourceName(model?.resource_name ?? "");
         setIsDefault(false);
     }, [open, model, type]);
 
+    // The error only describes the configuration it was tested against, so
+    // any change to the connection fields invalidates it.
+    useEffect(() => {
+        setConnectionError(null);
+    }, [adapter, providerModel, apiKey, url, resourceName]);
+
     const adapterOptions = adapterOptionsForType(type);
     const modelId = modelIdTouched ? modelIdInput : slugifyModelId(displayName);
     const showContextWindow = supportsContextWindow(type);
@@ -155,6 +165,42 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo
         (!requireUrl || url.trim().length > 0) &&
         (!requireResourceName || resourceName.trim().length > 0);
 
+    // On edit a save only needs a probe when connection config changed; a
+    // pure rename must not fail because the provider is briefly unreachable.
+    const connectionChanged =
+        !isEdit ||
+        !model ||
+        adapter !== model.adapter ||
+        providerModel.trim() !== model.provider_model ||
+        apiKey.trim().length > 0 ||
+        url.trim() !== (model.url ?? "") ||
+        resourceName.trim() !== (model.resource_name ?? "");
+
+    // Probes the connection with the current form values; on edit a blank API
+    // key means the stored key is used, referenced via model_id.
+    const verifyConnection = async (): Promise<boolean> => {
+        const result = await testModelConnection(apiClient, {
+            type,
+            adapter,
+            provider_model: providerModel.trim(),
+            credentials: {
+                ...(apiKey.trim() ? { apiKey: apiKey.trim() } : {}),
+                ...(url.trim() ? { url: url.trim() } : {}),
+                ...(resourceName.trim() ? { resourceName: resourceName.trim() } : {}),
+            },
+            ...(isEdit && model && !apiKey.trim() ? { model_id: model.model_id } : {}),
+        });
+
+        if (!result.ok) {
+            setConnectionError({
+                text: t(`settings.models.test.error.${result.reason}`),
+                detail: result.message,
+            });
+        }
+
+        return result.ok;
+    };
+
     const handleSubmit = async (e: React.FormEvent) => {
         e.preventDefault();
         if (showContextWindow && contextWindowValue === null) {
@@ -167,8 +213,12 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo
             return;
         }
         setContextWindowError(null);
+        setConnectionError(null);
         setLoading(true);
         try {
+            if (connectionChanged && !(await verifyConnection())) {
+                return;
+            }
             if (isEdit && model) {
                 const patch: ModelPatchInput = {};
                 if (displayName.trim() !== model.display_name) {
@@ -375,6 +425,17 @@ export function ModelFormDialog({ open, onOpenChange, type, model, onSaved }: Mo
                             <Switch id="model-is-default" checked={isDefault} onCheckedChange={setIsDefault} />
                         </div>
                     ) : null}
+                    {connectionError ? (
+                        <div className="space-y-1">
+                            <p className="flex items-center gap-1.5 text-xs text-destructive">
+                                <AlertCircle className="h-3.5 w-3.5 shrink-0" />
+                                {connectionError.text}
+                            </p>
+                            {connectionError.detail ? (
+                                <p className="text-xs text-muted-foreground break-words">{connectionError.detail}</p>
+                            ) : null}
+                        </div>
+                    ) : null}
                     <DialogFooter>
                         <Button type="button" variant="outline" onClick={() => onOpenChange(false)} disabled={loading}>
                             {t("cancel")}

apps/frontend/lib/api/models.ts

@@ -10,6 +10,8 @@ import type {
     ApiResponse,
     ModelCreateInput,
     ModelPatchInput,
+    ModelTestInput,
+    ModelTestResult,
     PublicModelListItem,
 } from "@kiwi/contracts";
 import { unwrapApiResponse, type KiwiApiClient } from "./client";
@@ -64,6 +66,15 @@ export async function updateModel(
     return unwrapApiResponse(response);
 }
 
+/**
+ * Probes the given configuration against the provider without persisting it.
+ * Omit credentials.apiKey and pass model_id to test with the stored key.
+ */
+export async function testModelConnection(client: KiwiApiClient, input: ModelTestInput): Promise<ModelTestResult> {
+    const response = await client.post<ApiResponse<ModelTestResult>>("/models/test", input);
+    return unwrapApiResponse(response);
+}
+
 /** Makes the AI Model the default for its type, clearing the previous default. */
 export async function setDefaultModel(client: KiwiApiClient, modelId: string): Promise<AdminModelListItem> {
     const response = await client.post<AdminModelResponse>(`${modelPath(modelId)}/default`);

apps/frontend/messages/de.json

@@ -443,6 +443,10 @@
     "settings.models.adapter.azure": "Azure OpenAI",
     "settings.models.adapter.anthropic": "Anthropic",
     "settings.models.adapter.openaiAPI": "OpenAI-kompatible API",
+    "settings.models.test.error.auth": "Authentifizierung fehlgeschlagen. Prüfe den API-Schlüssel.",
+    "settings.models.test.error.not_found": "Das Provider-Modell wurde nicht gefunden. Prüfe Provider-Modell-Name und Endpunkt.",
+    "settings.models.test.error.unreachable": "Der Anbieter ist nicht erreichbar. Prüfe Endpunkt-URL und Netzwerkzugriff.",
+    "settings.models.test.error.unknown": "Der Verbindungstest ist fehlgeschlagen.",
     "settings.models.created.renamed": "Als „{id}“ angelegt, da die gewünschte ID bereits vergeben oder normalisiert wurde.",
     "settings.models.error.invalid": "Die Modellkonfiguration ist für den gewählten Adapter ungültig.",
     "settings.models.delete.title": "KI-Modell löschen",

apps/frontend/messages/en.json

@@ -443,6 +443,10 @@
     "settings.models.adapter.azure": "Azure OpenAI",
     "settings.models.adapter.anthropic": "Anthropic",
     "settings.models.adapter.openaiAPI": "OpenAI-compatible API",
+    "settings.models.test.error.auth": "Authentication failed. Check the API key.",
+    "settings.models.test.error.not_found": "The provider model was not found. Check the provider model name and endpoint.",
+    "settings.models.test.error.unreachable": "The provider could not be reached. Check the endpoint URL and network access.",
+    "settings.models.test.error.unknown": "The connection test failed.",
     "settings.models.created.renamed": "Created as \"{id}\" because the requested ID was taken or normalized.",
     "settings.models.error.invalid": "The model configuration is invalid for the selected adapter.",
     "settings.models.delete.title": "Delete AI model",

packages/ai/package.json

@@ -9,6 +9,7 @@
         "./lock": "./src/lock.ts",
         "./mcp": "./src/mcp/index.ts",
         "./models": "./src/models.ts",
+        "./probe": "./src/probe.ts",
         "./tools/toolsets": "./src/tools/toolsets.ts",
         "./ui": "./src/ui.ts",
         "./prompts/*": "./src/prompts/*.ts"