From ebabfb8b50f1f0253e56301527e788775ba82ef2 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:09:05 -0700 Subject: [PATCH] Skip unmigratable orgs in the provider service split and fail boot loudly The boot-rail service split threw out of planMigration when a monolith config had no stored specHash, which failed the whole data-migration run and left the daemon wedged on startup with only 'Unknown error' on stderr (#1403). Route that throw through the existing per-org hardError path so the org is skipped, warned about, and retried on a later run, and warn about skipped orgs even when no org is migratable. A fatal CLI failure now exits the process instead of parking it with the port unbound, so the desktop shows the crash screen instead of an infinite 'Starting executor'. CLI error rendering walks the cause chain so wrapper errors print the underlying message instead of 'Unknown error'. --- .changeset/service-split-boot-resilience.md | 6 ++ apps/cli/src/main.ts | 50 +++++++++-- .../src/planner.test.ts | 83 +++++++++++++++++++ .../provider-service-split/src/planner.ts | 14 +++- .../provider-service-split/src/sqlite.test.ts | 53 ++++++++++++ .../provider-service-split/src/sqlite.ts | 16 ++-- 6 files changed, 208 insertions(+), 14 deletions(-) create mode 100644 .changeset/service-split-boot-resilience.md diff --git a/.changeset/service-split-boot-resilience.md b/.changeset/service-split-boot-resilience.md new file mode 100644 index 000000000..9c03648d3 --- /dev/null +++ b/.changeset/service-split-boot-resilience.md @@ -0,0 +1,6 @@ +--- +"@executor-js/plugin-provider-service-split": patch +"executor": patch +--- + +The provider service split boot migration now skips an org whose Google or Microsoft integration cannot be migrated (for example a config without a stored specHash) instead of failing the whole migration and blocking server startup. A daemon that does fail during boot now exits with the underlying error message instead of hanging with a generic "Unknown error". diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index d7046f086..bae18617c 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -1520,14 +1520,45 @@ const parseOptionalJsonObject = ( Effect.mapError((error) => new Error(`Invalid --content JSON: ${error.message}`)), ); +const ownMessage = (value: unknown): string => { + if (typeof value === "string") return value.trim(); + if (typeof value === "object" && value !== null && "message" in value) { + const message = (value as { message: unknown }).message; + if (typeof message === "string") return message.trim(); + } + return ""; +}; + +/** + * Render an unknown failure by walking its `cause` chain. Wrapper errors + * (tagged errors carrying only `{ cause }`) have an empty own message; without + * descending, a boot failure like a data-migration error printed as literally + * "Unknown error" (issue #1403). Nested messages that merely repeat their + * parent are dropped. + */ const formatUnknownMessage = (cause: unknown): string => { - if (cause instanceof Error) return cause.message; - if (typeof cause === "string") return cause; - if (typeof cause === "object" && cause !== null && "message" in cause) { - const message = cause.message; - if (typeof message === "string") return message; + const messages: string[] = []; + const seen = new Set(); + let current: unknown = cause; + while (current !== null && current !== undefined && !seen.has(current)) { + seen.add(current); + const message = ownMessage(current); + if (message.length > 0 && !messages.some((existing) => existing.includes(message))) { + messages.push(message); + } + current = + typeof current === "object" && "cause" in current + ? (current as { cause: unknown }).cause + : undefined; + } + if (messages.length === 0) { + return typeof cause === "string" + ? cause + : cause instanceof Error + ? cause.message + : String(cause); } - return String(cause); + return messages.join("\ncaused by: "); }; const readCliLogLevel = (argv: ReadonlyArray): string | undefined => { @@ -3205,7 +3236,12 @@ const program = ( } else { console.error(renderCliError(cause)); } - process.exitCode = 1; + // Exit hard, not via exitCode: this handler converts the failure into a + // success, so runMain's teardown never force-exits, and background fibers + // (the integrations-refresh fork) keep the event loop alive. A daemon + // that failed boot then lingers with its port unbound, and the desktop + // waits on the ready sentinel forever (issue #1403). + process.exit(1); }), ), ); diff --git a/packages/plugins/provider-service-split/src/planner.test.ts b/packages/plugins/provider-service-split/src/planner.test.ts index c21d85ea2..0a3773fe5 100644 --- a/packages/plugins/provider-service-split/src/planner.test.ts +++ b/packages/plugins/provider-service-split/src/planner.test.ts @@ -519,6 +519,89 @@ describe("provider service split migration planner", () => { expect(plan.summary.hardErrorOrgs).toBe(1); }); + it("records a hard error instead of throwing when a monolith has no specHash", () => { + const plan = planMigration( + input({ + integrations: [ + integration({ + config: { + googleDiscoveryUrls: [ + "https://www.googleapis.com/discovery/v1/apis/calendar/v3/rest", + ], + }, + }), + ], + collectPolicyErrors: true, + }), + ); + + expect(plan.orgs[0]?.hardErrors).toEqual([ + expect.stringContaining("has no specHash; serving state cannot be migrated"), + ]); + expect(plan.summary.hardErrorOrgs).toBe(1); + }); + + it("a specHash-less monolith only skips its own org", () => { + const plan = planMigration( + input({ + integrations: [ + integration({ + config: { + googleDiscoveryUrls: [ + "https://www.googleapis.com/discovery/v1/apis/calendar/v3/rest", + ], + }, + }), + integration({ tenant: "org_2", row_id: "int_2" }), + ], + connections: [connection(), connection({ tenant: "org_2", row_id: "conn_2" })], + tools: [ + tool("calendar.events.list"), + tool("calendar.events.list", { tenant: "org_2", row_id: "tool_2" }), + ], + pluginStorage: [ + operation("calendar.events.list"), + operation("calendar.events.list", { + tenant: "org_2", + row_id: "op_2", + }), + ], + blobs: [ + blob("spec/mono-hash"), + blob("defs/mono-hash"), + blob("spec/mono-hash", { namespace: "o:org_2/google", id: "blob_3" }), + blob("defs/mono-hash", { namespace: "o:org_2/google", id: "blob_4" }), + ], + collectPolicyErrors: true, + }), + ); + + const broken = plan.orgs.find((org) => org.tenant === "org_1"); + const healthy = plan.orgs.find((org) => org.tenant === "org_2"); + expect(broken?.hardErrors).toHaveLength(1); + expect(healthy?.hardErrors).toEqual([]); + expect(healthy?.integrations.map((row) => row.target.slug)).toEqual(["google_calendar"]); + expect(plan.summary.hardErrorOrgs).toBe(1); + }); + + it("still throws on a missing specHash outside collectPolicyErrors mode", () => { + expect(() => + planMigration( + input({ + integrations: [ + integration({ + config: { + googleDiscoveryUrls: [ + "https://www.googleapis.com/discovery/v1/apis/calendar/v3/rest", + ], + }, + }), + ], + }), + ), + ).toThrow(/has no specHash; serving state cannot be migrated/); + }); + it("retargets orphan block policies in boot-rail mode", () => { const plan = planMigration( input({ diff --git a/packages/plugins/provider-service-split/src/planner.ts b/packages/plugins/provider-service-split/src/planner.ts index eb7baf296..3e00af5f5 100644 --- a/packages/plugins/provider-service-split/src/planner.ts +++ b/packages/plugins/provider-service-split/src/planner.ts @@ -1045,7 +1045,19 @@ export const planMigration = (input: MigrationInput): MigrationPlan => { hardErrors.push(error instanceof Error ? error.message : String(error)); continue; } - const specHash = specHashFor(monolith); + // Same escape hatch as deriveServices above: under collectPolicyErrors a + // monolith without a usable specHash becomes a per-org hardError (org + // skipped, warned, left unstamped) instead of failing the whole plan. An + // escaped throw here took down every org in the run and, on the local + // boot rail, prevented the daemon from starting at all (issue #1403). + let specHash: string; + try { + specHash = specHashFor(monolith); + } catch (error) { + if (!input.collectPolicyErrors) throw error; + hardErrors.push(error instanceof Error ? error.message : String(error)); + continue; + } const namespace = pluginBlobNamespace(tenant, monolith.plugin_id); const targetNamespace = pluginBlobNamespace(tenant, "openapi"); const specBlobPresent = diff --git a/packages/plugins/provider-service-split/src/sqlite.test.ts b/packages/plugins/provider-service-split/src/sqlite.test.ts index 78a2327e2..fca87a64c 100644 --- a/packages/plugins/provider-service-split/src/sqlite.test.ts +++ b/packages/plugins/provider-service-split/src/sqlite.test.ts @@ -288,6 +288,59 @@ describe("providerServiceSplitDataMigration", () => { }), ); + it.effect("skips a specHash-less org intact and still migrates healthy orgs", () => + Effect.gen(function* () { + const db = yield* Effect.promise(() => createSqliteTestFumaDb({ tables: collectTables() })); + const client = db.client; + + // org_1: a real pre-split Google setup whose config never recorded a + // specHash (issue #1403 — this used to throw out of the planner and + // fail the whole boot migration). + yield* Effect.promise(() => + insertIntegration(client, { + rowId: "google_row_org_1", + tenant: "org_1", + slug: "google", + pluginId: "google", + config: { + googleDiscoveryUrls: ["https://www.googleapis.com/discovery/v1/apis/calendar/v3/rest"], + }, + }), + ); + yield* Effect.promise(() => insertConnection(client, "org_1")); + yield* Effect.promise(() => insertTool(client, "calendar.events.list", "org_1")); + yield* Effect.promise(() => insertOperation(client, "calendar.events.list", "org_1")); + yield* seedCalendarOrg(client, "org_2"); + + expect(yield* runSqliteDataMigrations(client, [providerServiceSplitDataMigration])).toEqual([ + "2026-07-08-provider-service-split", + ]); + + const integrations = yield* Effect.promise(() => + client.execute("SELECT tenant, slug, plugin_id FROM integration ORDER BY tenant, slug"), + ); + expect(integrations.rows).toEqual([ + { tenant: "org_1", slug: "google", plugin_id: "google" }, + { tenant: "org_2", slug: "google_calendar", plugin_id: "openapi" }, + ]); + + const ledger = yield* Effect.promise(() => + client.execute("SELECT tenant FROM provider_service_split_org_migration ORDER BY tenant"), + ); + expect(ledger.rows).toEqual([{ tenant: "org_2" }]); + + const connections = yield* Effect.promise(() => + client.execute("SELECT tenant, integration FROM connection ORDER BY tenant"), + ); + expect(connections.rows).toEqual([ + { tenant: "org_1", integration: "google" }, + { tenant: "org_2", integration: "google_calendar" }, + ]); + + yield* Effect.promise(() => db.close()); + }), + ); + it.effect("uses the per-org ledger to recover after a mid-run crash", () => Effect.gen(function* () { const db = yield* Effect.promise(() => createSqliteTestFumaDb({ tables: collectTables() })); diff --git a/packages/plugins/provider-service-split/src/sqlite.ts b/packages/plugins/provider-service-split/src/sqlite.ts index f87593043..37486b219 100644 --- a/packages/plugins/provider-service-split/src/sqlite.ts +++ b/packages/plugins/provider-service-split/src/sqlite.ts @@ -519,17 +519,21 @@ export const runSqliteProviderServiceSplitMigration = ( if (!(yield* tableExists(client, "integration"))) return 0; const input = yield* readDatabaseInput(client, options); const plan = planMigration(input); + // Warn about skipped orgs before the early return: when EVERY org has hard + // errors, moved is 0 and the loop below never runs, which made a fully + // skipped migration silent (issue #1403's failure mode after the fix). + for (const org of plan.orgs) { + if (org.completed || org.hardErrors.length === 0) continue; + console.warn( + `provider-service-split: skipped org ${org.tenantHash}: ${org.hardErrors.join("; ")}`, + ); + } const moved = plan.orgs.filter((org) => !org.completed && org.hardErrors.length === 0).length; if (moved === 0) return 0; for (const org of plan.orgs) { if (org.completed) continue; - if (org.hardErrors.length > 0) { - console.warn( - `provider-service-split: skipped org ${org.tenantHash}: ${org.hardErrors.join("; ")}`, - ); - continue; - } + if (org.hardErrors.length > 0) continue; yield* execute(client, "BEGIN"); const applyOne = Effect.gen(function* () { yield* applyOrg(client, org);