From a778ee8d17faaa757d2de3fc1581eb2746ddc579 Mon Sep 17 00:00:00 2001 From: Wistomize <48406052+Wistomize@users.noreply.github.com> Date: Thu, 16 Jul 2026 19:18:13 +0800 Subject: [PATCH] feat(agent): add governed analysis protocol runtime --- .env.example | 31 + .gitignore | 1 + apps/api/src/protocol-event-recovery.test.ts | 35 + apps/api/src/protocol-event-recovery.ts | 19 + apps/api/src/protocol-run-completion.test.ts | 114 ++ apps/api/src/protocol-run-completion.ts | 120 ++ apps/api/src/protocol-state-store.test.ts | 124 ++ apps/api/src/protocol-state-store.ts | 108 ++ apps/api/src/run-agent-assembly.ts | 17 + apps/api/src/run-checkpoint-projector.test.ts | 53 + apps/api/src/run-checkpoint-projector.ts | 11 + apps/api/src/run-finalizer.ts | 18 +- apps/api/src/run-input.test.ts | 30 + apps/api/src/run-input.ts | 24 + apps/api/src/server.ts | 44 +- package-lock.json | 20 + package.json | 2 + packages/agent-runtime/package.json | 1 + .../src/capabilities/action-router.test.ts | 403 ++++++ .../src/capabilities/action-router.ts | 312 +++++ .../capabilities/capability-registry.test.ts | 54 + .../src/capabilities/capability-registry.ts | 80 ++ .../tool-capability-plugin.test.ts | 52 + .../capabilities/tool-capability-plugin.ts | 40 + .../agent-runtime/src/capabilities/types.ts | 48 + .../src/config/agent-runtime-limits.ts | 234 ++++ .../src/context/inventory/context-limits.ts | 22 +- .../mastra/mastra-context-budget-processor.ts | 2 +- .../tool-observation-budget-profile.ts | 3 +- .../tool-observation-projection-policy.ts | 5 +- .../src/errors/tool-execution-error.ts | 248 ++++ packages/agent-runtime/src/index.ts | 339 ++++- .../src/protocol/analysis-contract.test.ts | 77 ++ .../src/protocol/analysis-contract.ts | 266 ++++ .../src/protocol/analysis-requirements.ts | 139 ++ .../src/protocol/definition-validator.test.ts | 73 ++ .../src/protocol/definition-validator.ts | 29 + .../in-memory-protocol-state-store.ts | 94 ++ .../model-analysis-contract-grounder.test.ts | 226 ++++ .../model-analysis-contract-grounder.ts | 309 +++++ ...del-analysis-requirement-extractor.test.ts | 175 +++ .../model-analysis-requirement-extractor.ts | 103 ++ .../model-protocol-classifier.test.ts | 35 + .../src/protocol/model-protocol-classifier.ts | 57 + .../protocol-handoff-coordinator.test.ts | 63 + .../protocol/protocol-handoff-coordinator.ts | 134 ++ .../src/protocol/protocol-handoff.test.ts | 35 + .../src/protocol/protocol-handoff.ts | 34 + .../src/protocol/protocol-public-api.test.ts | 35 + .../src/protocol/protocol-registry.ts | 27 + .../src/protocol/protocol-router.test.ts | 147 +++ .../src/protocol/protocol-router.ts | 133 ++ .../src/protocol/protocol-runtime.test.ts | 350 +++++ .../src/protocol/protocol-runtime.ts | 430 ++++++ .../src/protocol/protocols/data-analysis.ts | 780 +++++++++++ .../protocols/formal-protocols.test.ts | 488 +++++++ .../src/protocol/protocols/general-task.ts | 70 + .../src/protocol/result-verifier.test.ts | 212 +++ .../src/protocol/result-verifier.ts | 237 ++++ .../protocol/run-protocol-boundary.test.ts | 1159 +++++++++++++++++ .../src/protocol/run-protocol-boundary.ts | 729 +++++++++++ .../protocol/sql-semantic-validator.test.ts | 130 ++ .../src/protocol/sql-semantic-validator.ts | 247 ++++ packages/agent-runtime/src/protocol/types.ts | 115 ++ .../agent-runtime/src/runtime-limits.test.ts | 32 + packages/agent-runtime/src/runtime-limits.ts | 11 +- .../datalink-semantic-provider.test.ts | 78 ++ .../semantic/datalink-semantic-provider.ts | 97 ++ .../default-semantic-provider.test.ts | 34 + .../src/semantic/default-semantic-provider.ts | 26 + .../semantic/local-semantic-provider.test.ts | 58 + .../src/semantic/local-semantic-provider.ts | 36 + .../semantic/semantic-provider-chain.test.ts | 102 ++ .../src/semantic/semantic-provider-chain.ts | 93 ++ packages/agent-runtime/src/semantic/types.ts | 41 + packages/agent-runtime/src/testing.ts | 30 + .../src/tools/data-tools-cache.test.ts | 49 + .../agent-runtime/src/tools/data-tools.ts | 49 +- .../src/tools/governed-tool-factory.test.ts | 203 +++ .../src/tools/governed-tool-factory.ts | 66 +- .../src/tools/sql-dialect-validation.test.ts | 36 + .../src/tools/sql-dialect-validation.ts | 63 + packages/data-gateway/src/index.ts | 5 + packages/data-gateway/src/types.ts | 1 + .../conversation-message-repository.test.ts | 50 + packages/metadata/src/index.ts | 412 +++++- .../metadata/src/run-event-dedupe.test.ts | 39 + scripts/smoke-agent-protocol-deepseek.mjs | 215 +++ scripts/smoke-collaboration-tools.mjs | 2 +- scripts/smoke-datalink-semantic.mjs | 32 + scripts/smoke-protocol-recovery.mjs | 106 ++ scripts/smoke-run-finalizer.mjs | 19 + 92 files changed, 11672 insertions(+), 65 deletions(-) create mode 100644 apps/api/src/protocol-event-recovery.test.ts create mode 100644 apps/api/src/protocol-event-recovery.ts create mode 100644 apps/api/src/protocol-run-completion.test.ts create mode 100644 apps/api/src/protocol-run-completion.ts create mode 100644 apps/api/src/protocol-state-store.test.ts create mode 100644 apps/api/src/protocol-state-store.ts create mode 100644 apps/api/src/run-checkpoint-projector.test.ts create mode 100644 apps/api/src/run-input.test.ts create mode 100644 packages/agent-runtime/src/capabilities/action-router.test.ts create mode 100644 packages/agent-runtime/src/capabilities/action-router.ts create mode 100644 packages/agent-runtime/src/capabilities/capability-registry.test.ts create mode 100644 packages/agent-runtime/src/capabilities/capability-registry.ts create mode 100644 packages/agent-runtime/src/capabilities/tool-capability-plugin.test.ts create mode 100644 packages/agent-runtime/src/capabilities/tool-capability-plugin.ts create mode 100644 packages/agent-runtime/src/capabilities/types.ts create mode 100644 packages/agent-runtime/src/config/agent-runtime-limits.ts create mode 100644 packages/agent-runtime/src/errors/tool-execution-error.ts create mode 100644 packages/agent-runtime/src/protocol/analysis-contract.test.ts create mode 100644 packages/agent-runtime/src/protocol/analysis-contract.ts create mode 100644 packages/agent-runtime/src/protocol/analysis-requirements.ts create mode 100644 packages/agent-runtime/src/protocol/definition-validator.test.ts create mode 100644 packages/agent-runtime/src/protocol/definition-validator.ts create mode 100644 packages/agent-runtime/src/protocol/in-memory-protocol-state-store.ts create mode 100644 packages/agent-runtime/src/protocol/model-analysis-contract-grounder.test.ts create mode 100644 packages/agent-runtime/src/protocol/model-analysis-contract-grounder.ts create mode 100644 packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.test.ts create mode 100644 packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.ts create mode 100644 packages/agent-runtime/src/protocol/model-protocol-classifier.test.ts create mode 100644 packages/agent-runtime/src/protocol/model-protocol-classifier.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-handoff-coordinator.test.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-handoff-coordinator.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-handoff.test.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-handoff.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-public-api.test.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-registry.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-router.test.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-router.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-runtime.test.ts create mode 100644 packages/agent-runtime/src/protocol/protocol-runtime.ts create mode 100644 packages/agent-runtime/src/protocol/protocols/data-analysis.ts create mode 100644 packages/agent-runtime/src/protocol/protocols/formal-protocols.test.ts create mode 100644 packages/agent-runtime/src/protocol/protocols/general-task.ts create mode 100644 packages/agent-runtime/src/protocol/result-verifier.test.ts create mode 100644 packages/agent-runtime/src/protocol/result-verifier.ts create mode 100644 packages/agent-runtime/src/protocol/run-protocol-boundary.test.ts create mode 100644 packages/agent-runtime/src/protocol/run-protocol-boundary.ts create mode 100644 packages/agent-runtime/src/protocol/sql-semantic-validator.test.ts create mode 100644 packages/agent-runtime/src/protocol/sql-semantic-validator.ts create mode 100644 packages/agent-runtime/src/protocol/types.ts create mode 100644 packages/agent-runtime/src/runtime-limits.test.ts create mode 100644 packages/agent-runtime/src/semantic/datalink-semantic-provider.test.ts create mode 100644 packages/agent-runtime/src/semantic/datalink-semantic-provider.ts create mode 100644 packages/agent-runtime/src/semantic/default-semantic-provider.test.ts create mode 100644 packages/agent-runtime/src/semantic/default-semantic-provider.ts create mode 100644 packages/agent-runtime/src/semantic/local-semantic-provider.test.ts create mode 100644 packages/agent-runtime/src/semantic/local-semantic-provider.ts create mode 100644 packages/agent-runtime/src/semantic/semantic-provider-chain.test.ts create mode 100644 packages/agent-runtime/src/semantic/semantic-provider-chain.ts create mode 100644 packages/agent-runtime/src/semantic/types.ts create mode 100644 packages/agent-runtime/src/tools/data-tools-cache.test.ts create mode 100644 packages/agent-runtime/src/tools/governed-tool-factory.test.ts create mode 100644 packages/agent-runtime/src/tools/sql-dialect-validation.test.ts create mode 100644 packages/agent-runtime/src/tools/sql-dialect-validation.ts create mode 100644 packages/metadata/src/conversation-message-repository.test.ts create mode 100644 packages/metadata/src/run-event-dedupe.test.ts create mode 100644 scripts/smoke-agent-protocol-deepseek.mjs create mode 100644 scripts/smoke-datalink-semantic.mjs create mode 100644 scripts/smoke-protocol-recovery.mjs diff --git a/.env.example b/.env.example index 133c216..593c757 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,37 @@ LLM_MODEL=qwen-plus LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 LLM_API_KEY= +# Optional Agent runtime limit overrides. Defaults and safe ranges are defined in +# packages/agent-runtime/src/config/agent-runtime-limits.ts. +# DATAFOUNDRY_AGENT_MAX_STEPS=80 +# DATAFOUNDRY_SQL_MAX_EXECUTION_COUNT=60 +# DATAFOUNDRY_DATA_ANALYSIS_MAX_PROTOCOL_ACTIONS=500 +# DATAFOUNDRY_GENERAL_TASK_MAX_PROTOCOL_ACTIONS=100 +# DATAFOUNDRY_PROTOCOL_DEFAULT_MAX_ACTIONS=100 +# DATAFOUNDRY_PROTOCOL_MAX_COMPLETION_REJECTIONS=3 +# DATAFOUNDRY_PROTOCOL_MAX_COMMIT_RETRIES=8 +# DATAFOUNDRY_PROTOCOL_AUTOMATIC_ACTION_MAX_DEPTH=10 +# DATAFOUNDRY_SCHEMA_MAX_TABLES=20 +# DATAFOUNDRY_SCHEMA_MAX_COLUMNS_PER_TABLE=50 +# DATAFOUNDRY_SQL_MAX_MODEL_ROWS=20 +# DATAFOUNDRY_SQL_MAX_ACTIVITY_ROWS=20 +# DATAFOUNDRY_SQL_MAX_CELL_CHARS=500 +# DATAFOUNDRY_SQL_MAX_SQL_CHARS=4000 +# DATAFOUNDRY_CONTEXT_MAX_TOKENS=32000 +# DATAFOUNDRY_CONTEXT_MAX_CHARS=32000 +# DATAFOUNDRY_TOOL_ERROR_MAX_MESSAGE_CHARS=500 +# DATAFOUNDRY_KNOWLEDGE_MAX_TOP_K=20 +# DATAFOUNDRY_REQUIREMENT_COMMIT_MAX_CLAIMS=16 +# DATAFOUNDRY_REQUIREMENT_COMMIT_MAX_OUTPUT_FIELDS=32 +# DATAFOUNDRY_MODEL_HELPER_MAX_STEPS=1 +# DATAFOUNDRY_PROTOCOL_CLASSIFIER_MAX_OUTPUT_TOKENS=512 +# DATAFOUNDRY_REQUIREMENT_EXTRACTOR_MAX_OUTPUT_TOKENS=4096 +# DATAFOUNDRY_CONTRACT_GROUNDER_MAX_OUTPUT_TOKENS=8192 +# DATAFOUNDRY_CONTRACT_GROUNDER_MAX_ATTEMPTS=2 +# DATAFOUNDRY_TOOL_OBSERVATION_MAX_NAMES=5 +# DATAFOUNDRY_TOOL_OBSERVATION_MAX_NAME_CHARS=120 +# DATAFOUNDRY_TOOL_OBSERVATION_MAX_CHARS=12000 + EMBEDDING_PROVIDER=bailian EMBEDDING_MODEL=text-embedding-v4 EMBEDDING_DIM=1024 diff --git a/.gitignore b/.gitignore index f9120d1..277b382 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ site/ .next/ **/.next/ coverage/ +artifacts/ storage/ **/storage/ .cache/ diff --git a/apps/api/src/protocol-event-recovery.test.ts b/apps/api/src/protocol-event-recovery.test.ts new file mode 100644 index 0000000..83400b6 --- /dev/null +++ b/apps/api/src/protocol-event-recovery.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { replayPendingProtocolEvents } from "./protocol-event-recovery.js"; + +describe("replayPendingProtocolEvents", () => { + it("acknowledges a journal event only after delivery succeeds", () => { + const event = { + eventId: "event-1", + type: "protocol.state.updated", + runId: "run-1", + segmentId: "run-1:segment:1", + protocolId: "general-task", + protocolVersion: "1", + revision: 2 + }; + const delivered: unknown[] = []; + const acknowledged: unknown[] = []; + + replayPendingProtocolEvents({ + runId: "run-1", + stateStore: { + pendingEvents: () => [event], + acknowledgeEvent: (pending) => acknowledged.push(pending) + }, + emit: (pending) => delivered.push(pending) + }); + + expect(delivered).toMatchObject([{ + type: "CUSTOM", + name: "protocol.state.updated", + value: event + }]); + expect(acknowledged).toEqual([event]); + }); +}); diff --git a/apps/api/src/protocol-event-recovery.ts b/apps/api/src/protocol-event-recovery.ts new file mode 100644 index 0000000..5290c1d --- /dev/null +++ b/apps/api/src/protocol-event-recovery.ts @@ -0,0 +1,19 @@ +import type { BaseEvent } from "@ag-ui/client"; +import { createCustomEvent, type ProtocolEvent } from "@datafoundry/agent-runtime"; + +type ProtocolEventJournal = { + acknowledgeEvent(event: ProtocolEvent): void; + pendingEvents(runId: string): ProtocolEvent[]; +}; + +/** Deliver durable protocol journal entries that were not published before a previous process stopped. */ +export const replayPendingProtocolEvents = (input: { + runId: string; + stateStore: ProtocolEventJournal; + emit(event: BaseEvent): void; +}): void => { + for (const event of input.stateStore.pendingEvents(input.runId)) { + input.emit(createCustomEvent(event.type, event)); + input.stateStore.acknowledgeEvent(event); + } +}; diff --git a/apps/api/src/protocol-run-completion.test.ts b/apps/api/src/protocol-run-completion.test.ts new file mode 100644 index 0000000..301da03 --- /dev/null +++ b/apps/api/src/protocol-run-completion.test.ts @@ -0,0 +1,114 @@ +import { EventType, type BaseEvent } from "@ag-ui/client"; +import type { ProtocolRunState } from "@datafoundry/agent-runtime"; +import { describe, expect, it, vi } from "vitest"; + +import { assistantMessageIdFromEvent, completeProtocolRun } from "./protocol-run-completion.js"; + +const terminalEvent = { type: EventType.RUN_FINISHED, timestamp: 1 } as BaseEvent; + +describe("completeProtocolRun", () => { + it("recognizes a tool-only assistant turn through its parent message id", () => { + expect(assistantMessageIdFromEvent({ + type: EventType.TOOL_CALL_START, + toolCallId: "tool-1", + toolCallName: "write_file", + parentMessageId: "tool-parent-message" + } as BaseEvent)).toBe("tool-parent-message"); + }); + + it("uses the latest persisted assistant message when the current segment has no text event", async () => { + const harness = createHarness({}); + + await completeProtocolRun({ + ...harness.input, + persistedAssistantMessageId: "persisted-message", + terminalEvent + }); + + expect(harness.execute).toHaveBeenCalledWith(expect.objectContaining({ + actionName: "general.answer.commit", + input: { messageId: "persisted-message" } + })); + expect(harness.complete).toHaveBeenCalledOnce(); + }); + + it("does not submit a second answer after the protocol has entered the answer phase", async () => { + const harness = createHarness({ answerMessageId: "committed-message", phase: "answer" }); + + await completeProtocolRun({ + ...harness.input, + lastAssistantMessageId: "committed-message", + terminalEvent + }); + + expect(harness.execute).not.toHaveBeenCalled(); + expect(harness.complete).toHaveBeenCalledOnce(); + }); + + it("emits a clean run error when terminal protocol finalization fails", async () => { + const harness = createHarness({}); + harness.execute.mockRejectedValueOnce(new Error("ACTION_NOT_ALLOWED_IN_PHASE:answer:general.answer.commit")); + + await expect(completeProtocolRun({ + ...harness.input, + lastAssistantMessageId: "message-1", + terminalEvent + })).resolves.toBeUndefined(); + + expect(harness.fail).toHaveBeenCalledWith({ + errorMessage: "ACTION_NOT_ALLOWED_IN_PHASE:answer:general.answer.commit", + terminalEvent: expect.objectContaining({ + type: EventType.RUN_ERROR, + message: "ACTION_NOT_ALLOWED_IN_PHASE:answer:general.answer.commit" + }) + }); + }); +}); + +const createHarness = (input: { answerMessageId?: string; phase?: string }) => { + const execute = vi.fn(async () => undefined); + const complete = vi.fn(async () => undefined); + const fail = vi.fn(); + let state: ProtocolRunState = { + protocolId: "general-task", + protocolVersion: "1", + runId: "run-1", + segmentId: "segment-1", + phase: input.phase ?? "gather", + revision: 1, + status: "active", + contextPackageRef: { packageId: "context-1", revision: 1 }, + actions: [], + completionRejections: 0, + domain: input.answerMessageId ? { answerMessageId: input.answerMessageId } : {}, + }; + const protocolRuntime = { + getState: vi.fn(() => state), + proposeCompletion: vi.fn(() => { + state = { + ...state, + revision: state.revision + 1, + terminalDecision: { + status: "completed", + evaluatedContextPackageRef: { packageId: "context-1", revision: 1 }, + evidenceRefs: [] + } + }; + return state; + }) + }; + execute.mockImplementation(async () => { + state = { ...state, phase: "answer", domain: { answerMessageId: "persisted-message" } }; + return undefined; + }); + return { + complete, + execute, + fail, + input: { + finalizer: { complete, fail }, + protocol: { actionRouter: { execute }, protocolRuntime, segmentId: "segment-1" }, + runId: "run-1" + } + }; +}; diff --git a/apps/api/src/protocol-run-completion.ts b/apps/api/src/protocol-run-completion.ts new file mode 100644 index 0000000..30f06d0 --- /dev/null +++ b/apps/api/src/protocol-run-completion.ts @@ -0,0 +1,120 @@ +import { EventType, type BaseEvent } from "@ag-ui/client"; +import type { ProtocolRunState } from "@datafoundry/agent-runtime"; + +import type { RunFinalizer } from "./run-finalizer.js"; + +type ProtocolCompletionInput = { + finalizer: Pick; + goalRuntime?: Parameters[0]["goalRuntime"]; + lastAssistantMessageId?: string; + persistedAssistantMessageId?: string; + protocol: { + actionRouter: { + execute(input: { + runId: string; + segmentId: string; + actionId: string; + actionName: string; + input: unknown; + idempotencyKey?: string; + }): Promise; + }; + protocolRuntime: { + getState(runId: string, segmentId?: string): ProtocolRunState; + proposeCompletion(input: { + runId: string; + segmentId: string; + expectedRevision: number; + forceTerminal?: boolean; + }): ProtocolRunState; + }; + segmentId: string; + }; + runId: string; + terminalEvent: BaseEvent; +}; + +/** Complete a governed protocol run and convert finalization failures into a durable RUN_ERROR. */ +export const completeProtocolRun = async (input: ProtocolCompletionInput): Promise => { + try { + let protocolState = input.protocol.protocolRuntime.getState(input.runId, input.protocol.segmentId); + const answerMessageId = input.lastAssistantMessageId ?? input.persistedAssistantMessageId; + if ( + protocolState.protocolId === "general-task" + && answerMessageId + && !committedGeneralAnswerMessageId(protocolState) + ) { + await input.protocol.actionRouter.execute({ + runId: input.runId, + segmentId: input.protocol.segmentId, + actionId: `${input.runId}:general-answer-commit`, + actionName: "general.answer.commit", + input: { messageId: answerMessageId }, + idempotencyKey: answerMessageId + }); + protocolState = input.protocol.protocolRuntime.getState(input.runId, input.protocol.segmentId); + } + const terminalState = input.protocol.protocolRuntime.proposeCompletion({ + runId: input.runId, + segmentId: input.protocol.segmentId, + expectedRevision: protocolState.revision, + forceTerminal: true + }); + const terminalDecision = terminalState.terminalDecision; + if (!terminalDecision) { + throw new Error("PROTOCOL_TERMINAL_DECISION_REQUIRED"); + } + if (terminalDecision.status === "failed") { + input.finalizer.fail({ + errorMessage: terminalDecision.reasons.join("; "), + terminalEvent: createRunErrorEvent(terminalDecision.reasons.join("; ")) + }); + return; + } + await input.finalizer.complete({ + ...(input.goalRuntime ? { goalRuntime: input.goalRuntime } : {}), + terminalDecision, + terminalEvent: input.terminalEvent + }); + } catch (error) { + const message = error instanceof Error ? error.message : "PROTOCOL_FINALIZATION_FAILED"; + input.finalizer.fail({ errorMessage: message, terminalEvent: createRunErrorEvent(message) }); + } +}; + +/** Return the assistant message identifier carried by text or tool-parent AG-UI events. */ +export const assistantMessageIdFromEvent = (event: BaseEvent): string | undefined => { + if (event.type === EventType.TOOL_CALL_START) { + const parentMessageId = (event as BaseEvent & { parentMessageId?: string }).parentMessageId; + return typeof parentMessageId === "string" && parentMessageId.length > 0 ? parentMessageId : undefined; + } + if ( + event.type !== EventType.TEXT_MESSAGE_START + && event.type !== EventType.TEXT_MESSAGE_CONTENT + && event.type !== EventType.TEXT_MESSAGE_CHUNK + && event.type !== EventType.TEXT_MESSAGE_END + ) { + return undefined; + } + const candidate = event as BaseEvent & { messageId?: string; role?: string }; + if (candidate.role && candidate.role !== "assistant") { + return undefined; + } + return typeof candidate.messageId === "string" && candidate.messageId.length > 0 + ? candidate.messageId + : undefined; +}; + +const committedGeneralAnswerMessageId = (state: ProtocolRunState): string | undefined => { + if (typeof state.domain !== "object" || state.domain === null || Array.isArray(state.domain)) { + return undefined; + } + const messageId = (state.domain as Record).answerMessageId; + return typeof messageId === "string" && messageId.length > 0 ? messageId : undefined; +}; + +const createRunErrorEvent = (message: string): BaseEvent => ({ + type: EventType.RUN_ERROR, + message, + timestamp: Date.now() +}); diff --git a/apps/api/src/protocol-state-store.test.ts b/apps/api/src/protocol-state-store.test.ts new file mode 100644 index 0000000..4bedc6e --- /dev/null +++ b/apps/api/src/protocol-state-store.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createMetadataStore } from "@datafoundry/metadata"; +import { MetadataProtocolStateStore } from "./protocol-state-store.js"; + +describe("MetadataProtocolStateStore", () => { + it("restores a persisted protocol segment", () => { + const root = mkdtempSync(join(tmpdir(), "protocol-state-store-")); + try { + const metadata = createMetadataStore({ database_path: join(root, "metadata.sqlite") }); + metadata.sessions.create({ user_id: "dev-user", id: "session-1", title: "Protocol" }); + metadata.runs.create({ + user_id: "dev-user", + id: "run-1", + session_id: "session-1", + user_input: "test", + status: "running" + }); + metadata.contextPackageSnapshots.create({ + user_id: "dev-user", + session_id: "session-1", + run_id: "run-1", + package_id: "context-1", + revision: 0, + payload: {} + }); + const store = new MetadataProtocolStateStore(metadata, "dev-user"); + const startedEvent = { + eventId: "run-1:segment:1:0:protocol.run.started", + type: "protocol.run.started", + runId: "run-1", + segmentId: "run-1:segment:1", + protocolId: "general-task", + protocolVersion: "1", + revision: 0 + }; + store.create({ + protocolId: "general-task", + protocolVersion: "1", + runId: "run-1", + segmentId: "run-1:segment:1", + revision: 0, + phase: "work", + status: "active", + contextPackageRef: { packageId: "context-1", revision: 0 }, + actions: [], + completionRejections: 0, + domain: {} + }, [startedEvent]); + + const restored = new MetadataProtocolStateStore(metadata, "dev-user") + .get("run-1", "run-1:segment:1"); + expect(restored).toMatchObject({ protocolId: "general-task", revision: 0, phase: "work" }); + expect(store.pendingEvents("run-1")).toEqual([startedEvent]); + store.acknowledgeEvent(startedEvent); + expect(store.pendingEvents("run-1")).toEqual([]); + metadata.close(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("persists a handoff as one segment transition", () => { + const root = mkdtempSync(join(tmpdir(), "protocol-state-handoff-")); + try { + const metadata = createMetadataStore({ database_path: join(root, "metadata.sqlite") }); + metadata.sessions.create({ user_id: "dev-user", id: "session-1", title: "Protocol" }); + metadata.runs.create({ + user_id: "dev-user", + id: "run-1", + session_id: "session-1", + user_input: "test", + status: "running" + }); + metadata.contextPackageSnapshots.create({ + user_id: "dev-user", + session_id: "session-1", + run_id: "run-1", + package_id: "context-1", + revision: 0, + payload: {} + }); + const store = new MetadataProtocolStateStore(metadata, "dev-user"); + const current = store.create(createState("general-task", "run-1:segment:1", 0, "active")); + + store.transitionSegment({ + current: { ...current, revision: 1, status: "handed_off" }, + expectedRevision: 0, + next: createState("data-analysis", "run-1:segment:2", 0, "active") + }); + + expect(store.get("run-1", "run-1:segment:1").status).toBe("handed_off"); + expect(store.get("run-1")).toMatchObject({ + protocolId: "data-analysis", + segmentId: "run-1:segment:2" + }); + metadata.close(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +const createState = ( + protocolId: string, + segmentId: string, + revision: number, + status: "active" | "handed_off" +) => ({ + protocolId, + protocolVersion: "1", + runId: "run-1", + segmentId, + revision, + phase: "work", + status, + contextPackageRef: { packageId: "context-1", revision: 0 }, + actions: [], + completionRejections: 0, + domain: {} +}); diff --git a/apps/api/src/protocol-state-store.ts b/apps/api/src/protocol-state-store.ts new file mode 100644 index 0000000..64905b1 --- /dev/null +++ b/apps/api/src/protocol-state-store.ts @@ -0,0 +1,108 @@ +import type { ProtocolEvent, ProtocolRunState, ProtocolStateStore } from "@datafoundry/agent-runtime"; +import type { MetadataStore, ProtocolStateSnapshotRecord } from "@datafoundry/metadata"; + +/** Persist Protocol Runtime snapshots through the user-scoped metadata repository. */ +export class MetadataProtocolStateStore implements ProtocolStateStore { + constructor( + private readonly metadataStore: MetadataStore, + private readonly userId: string + ) {} + + create( + state: ProtocolRunState, + events: ProtocolEvent[] = [] + ): ProtocolRunState { + return this.persist(state, -1, events); + } + + find(runId: string, segmentId?: string): ProtocolRunState | undefined { + const record = segmentId + ? this.metadataStore.protocolStates.find({ + user_id: this.userId, + run_id: runId, + segment_id: segmentId + }) + : this.metadataStore.protocolStates.latestByRun({ user_id: this.userId, run_id: runId }); + return record ? parseProtocolState(record) : undefined; + } + + get(runId: string, segmentId?: string): ProtocolRunState { + const state = this.find(runId, segmentId); + if (!state) { + throw new Error(`PROTOCOL_RUN_NOT_FOUND:${runId}${segmentId ? `:${segmentId}` : ""}`); + } + return state; + } + + compareAndSet( + state: ProtocolRunState, + expectedRevision: number, + events: ProtocolEvent[] = [] + ): ProtocolRunState { + return this.persist(state, expectedRevision, events); + } + + transitionSegment(input: { + current: ProtocolRunState; + expectedRevision: number; + next: ProtocolRunState; + events?: ProtocolEvent[]; + }): { + current: ProtocolRunState; + next: ProtocolRunState; + } { + const records = this.metadataStore.protocolStates.transitionSegment({ + user_id: this.userId, + current: { + run_id: input.current.runId, + segment_id: input.current.segmentId, + expected_revision: input.expectedRevision, + state: input.current + }, + next: { + run_id: input.next.runId, + segment_id: input.next.segmentId, + expected_revision: -1, + state: input.next + }, + events: input.events ?? [] + }); + return { + current: parseProtocolState(records.current), + next: parseProtocolState(records.next) + }; + } + + acknowledgeEvent(event: ProtocolEvent): void { + this.metadataStore.protocolStates.acknowledgeEvent({ + user_id: this.userId, + event_id: event.eventId + }); + } + + pendingEvents(runId: string): ProtocolEvent[] { + return this.metadataStore.protocolStates.pendingEvents({ + user_id: this.userId, + run_id: runId + }) as ProtocolEvent[]; + } + + private persist( + state: ProtocolRunState, + expectedRevision: number, + events: ProtocolEvent[] + ): ProtocolRunState { + const record = this.metadataStore.protocolStates.compareAndSetWithEvents({ + user_id: this.userId, + run_id: state.runId, + segment_id: state.segmentId, + expected_revision: expectedRevision, + state + }, events); + return parseProtocolState(record); + } +} + +const parseProtocolState = ( + record: ProtocolStateSnapshotRecord +): ProtocolRunState => JSON.parse(record.state_json) as ProtocolRunState; diff --git a/apps/api/src/run-agent-assembly.ts b/apps/api/src/run-agent-assembly.ts index 8ebe93c..a5d56e7 100644 --- a/apps/api/src/run-agent-assembly.ts +++ b/apps/api/src/run-agent-assembly.ts @@ -10,6 +10,9 @@ import { type ContextPackage, type ContextPackageRecorder, type GoalRuntimeAdapter, + type RunProtocolBoundary, + type ContextPackageRef, + type ProtocolStateStore, type TaskStateRuntime, type WorkspaceAttachment } from "@datafoundry/agent-runtime"; @@ -29,6 +32,8 @@ export type RunAgentAssembly = { goalRuntime?: GoalRuntimeAdapter | undefined; governedMessages: RunAgentInput["messages"]; mastraAgent: MastraAgent; + protocol: RunProtocolBoundary; + flushProtocolEvents(): void; workspace: { command_execution_enabled: boolean; isolation: "bwrap" | "none" | "seatbelt"; @@ -57,6 +62,7 @@ type CreateRunAgentAssemblyInput = { effectiveRunConfig: EffectiveRunConfig; emitter: AgUiEventEmitter; contextPackageRecorder?: ContextPackageRecorder; + contextPackageExists(reference: ContextPackageRef): boolean; evidenceContextItems?: AgentContextItem[] | undefined; fileAssetService: FileAssetService; goal?: EffectiveRunConfig["goal"] | undefined; @@ -68,6 +74,7 @@ type CreateRunAgentAssemblyInput = { messages: RunAgentInput["messages"]; modelContextProfile?: ResolvedRunConfig["modelContextProfile"] | undefined; modelProvider: ResolvedRunConfig["modelProvider"]; + protocolStateStore: ProtocolStateStore; modelSettings?: ResolvedRunConfig["modelSettings"] | undefined; runContext: AgentRunContext; sessionOutputService: SessionOutputService; @@ -134,13 +141,16 @@ export const createRunAgentAssembly = async ( destroyWorkspace, goalRuntime, governedMessages, + flushProtocolEvents, isolation, + protocol, workspaceDir, sessionDir } = await createDataFoundry({ ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), artifactService: input.artifactService, ...(input.contextPackageRecorder ? { contextPackageRecorder: input.contextPackageRecorder } : {}), + contextPackageExists: input.contextPackageExists, dataGateway: input.dataGateway, fileAssetService: input.fileAssetService, ...(input.initialContextPackage ? { initialContextPackage: input.initialContextPackage } : {}), @@ -148,9 +158,14 @@ export const createRunAgentAssembly = async ( ...(input.mcpRuntime.toolNames.length > 0 ? { mcpToolNames: input.mcpRuntime.toolNames } : {}), ...(Object.keys(mcpTools).length > 0 ? { mcpTools } : {}), emitter: input.emitter, + ...(input.effectiveRunConfig.protocol ? { explicitProtocol: input.effectiveRunConfig.protocol } : {}), messages: input.messages, ...(input.modelContextProfile ? { modelContextProfile: input.modelContextProfile } : {}), modelProvider: input.modelProvider, + protocolStateStore: input.protocolStateStore, + ...(input.effectiveRunConfig.resourceRevisions + ? { resourceRevisions: input.effectiveRunConfig.resourceRevisions } + : {}), ...(input.modelSettings ? { modelSettings: input.modelSettings } : {}), ...(input.evidenceContextItems?.length ? { evidenceContextItems: input.evidenceContextItems } : {}), ...(input.longTermMemories.length > 0 ? { longTermMemory: { records: input.longTermMemories } } : {}), @@ -172,9 +187,11 @@ export const createRunAgentAssembly = async ( return { destroyWorkspace, + flushProtocolEvents, ...(goalRuntime ? { goalRuntime } : {}), governedMessages, mastraAgent, + protocol, workspace: { command_execution_enabled: commandExecutionEnabled, isolation diff --git a/apps/api/src/run-checkpoint-projector.test.ts b/apps/api/src/run-checkpoint-projector.test.ts new file mode 100644 index 0000000..eecd7af --- /dev/null +++ b/apps/api/src/run-checkpoint-projector.test.ts @@ -0,0 +1,53 @@ +import { createCustomEvent } from "@datafoundry/agent-runtime"; +import { createMetadataStore, RunEventWriter } from "@datafoundry/metadata"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { RunCheckpointProjector } from "./run-checkpoint-projector.js"; + +describe("RunCheckpointProjector protocol events", () => { + it("projects protocol phase entry against the referenced latest ContextPackage", () => { + const root = mkdtempSync(join(tmpdir(), "protocol-checkpoint-")); + try { + const metadata = createMetadataStore({ database_path: join(root, "metadata.sqlite") }); + metadata.sessions.create({ user_id: "dev-user", id: "session-1", title: "Protocol" }); + metadata.runs.create({ + user_id: "dev-user", + id: "run-1", + session_id: "session-1", + user_input: "test", + status: "running" + }); + metadata.contextPackageSnapshots.create({ + user_id: "dev-user", + session_id: "session-1", + run_id: "run-1", + package_id: "context-1", + revision: 2, + payload: {} + }); + const envelope = new RunEventWriter(metadata.runEvents).write({ + user_id: "dev-user", + run_id: "run-1", + session_id: "session-1", + event: createCustomEvent("protocol.phase.entered", { + eventId: "event-1", + payload: { phase: "query_planning" } + }) + }); + + new RunCheckpointProjector(metadata, "dev-user").observe(envelope); + + expect(metadata.checkpoints.latestByRun({ user_id: "dev-user", run_id: "run-1" })).toMatchObject({ + kind: "protocol-phase", + label: "Protocol phase: query_planning", + context_package_revision: 2 + }); + metadata.close(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/api/src/run-checkpoint-projector.ts b/apps/api/src/run-checkpoint-projector.ts index 6b00d1c..a822eb6 100644 --- a/apps/api/src/run-checkpoint-projector.ts +++ b/apps/api/src/run-checkpoint-projector.ts @@ -20,6 +20,17 @@ export class RunCheckpointProjector { return; } + if (event.type === EventType.CUSTOM && event.name === "protocol.phase.entered") { + const value = recordValue(event.value); + const payload = recordValue(value?.payload); + this.createLatestCheckpoint(envelope, { + kind: "protocol-phase", + label: `Protocol phase: ${stringValue(payload?.phase) ?? "unknown"}`, + status: "stable" + }); + return; + } + if (event.type === EventType.TOOL_CALL_RESULT) { this.createLatestCheckpoint(envelope, { kind: "tool-result", diff --git a/apps/api/src/run-finalizer.ts b/apps/api/src/run-finalizer.ts index a3a13a3..37d3487 100644 --- a/apps/api/src/run-finalizer.ts +++ b/apps/api/src/run-finalizer.ts @@ -1,5 +1,9 @@ import { EventType, type BaseEvent } from "@ag-ui/client"; -import { createCustomEvent, type GoalRuntimeAdapter } from "@datafoundry/agent-runtime"; +import { + createCustomEvent, + type GoalRuntimeAdapter, + type ProtocolCompletionDecision +} from "@datafoundry/agent-runtime"; import type { FileAssetService } from "@datafoundry/files"; import { readdirSync, statSync } from "node:fs"; import { join, relative, sep } from "node:path"; @@ -70,7 +74,17 @@ export class RunFinalizer { this.input.emit(input.terminalEvent); } - async complete(input: { goalRuntime?: GoalRuntimeAdapter | undefined; terminalEvent: BaseEvent }): Promise { + async complete(input: { + goalRuntime?: GoalRuntimeAdapter | undefined; + terminalDecision?: ProtocolCompletionDecision | undefined; + terminalEvent: BaseEvent; + }): Promise { + if (!input.terminalDecision || input.terminalDecision.status === "continue") { + throw new Error("PROTOCOL_TERMINAL_DECISION_REQUIRED"); + } + if (input.terminalDecision.status === "failed") { + throw new Error("PROTOCOL_FAILED_DECISION_REQUIRES_FAIL_FINALIZATION"); + } if (input.goalRuntime) { this.input.emit(createCustomEvent("goal.updated", { goal: await input.goalRuntime.getSnapshot(), diff --git a/apps/api/src/run-input.test.ts b/apps/api/src/run-input.test.ts new file mode 100644 index 0000000..f8e2735 --- /dev/null +++ b/apps/api/src/run-input.test.ts @@ -0,0 +1,30 @@ +import type { RunAgentInput } from "@ag-ui/client"; +import { describe, expect, it } from "vitest"; + +import { extractEffectiveRunConfig } from "./run-input.js"; + +describe("extractEffectiveRunConfig protocol selection", () => { + it("parses an explicit protocol identity from run_config", () => { + const config = extractEffectiveRunConfig(createInput({ + protocol: { id: "data-analysis", version: "1" } + })); + + expect(config.protocol).toEqual({ protocolId: "data-analysis", protocolVersion: "1" }); + }); + + it("rejects a partially specified explicit protocol", () => { + expect(() => extractEffectiveRunConfig(createInput({ + protocol: { id: "data-analysis" } + }))).toThrow("INVALID_PROTOCOL_SELECTION"); + }); +}); + +const createInput = (runConfig: Record): RunAgentInput => ({ + context: [], + forwardedProps: { run_config: runConfig }, + messages: [], + runId: "run-1", + state: {}, + threadId: "thread-1", + tools: [] +}); diff --git a/apps/api/src/run-input.ts b/apps/api/src/run-input.ts index ded31e9..ed45dee 100644 --- a/apps/api/src/run-input.ts +++ b/apps/api/src/run-input.ts @@ -50,6 +50,10 @@ export type EffectiveRunConfig = { pinnedPaths?: string[]; /** User-selected evidence references for this run. Resolved server-side before prompt assembly. */ evidenceRefs: EvidenceRef[]; + protocol?: { + protocolId: string; + protocolVersion: string; + }; /** * Resources silently dropped from `enabled*Ids` because `default_enabled=false` (R-020). * The run continues; this list is surfaced in `run.config.resolved` for diagnostics. @@ -123,6 +127,7 @@ export const extractEffectiveRunConfig = ( // R-024: parse pinned session-relative paths. Drop anything that escapes or is unsafe. const pinnedPaths = pinnedPathsFromAliases(runConfig, ["pinnedPaths", "pinned_paths"]); const evidenceRefs = evidenceRefsFromAliases(runConfig, ["evidenceRefs", "evidence_refs"]); + const protocol = protocolSelectionFromRunConfig(runConfig); if ( activeDatasourceId @@ -151,6 +156,7 @@ export const extractEffectiveRunConfig = ( ...(goal ? { goal } : {}), ...(mentioned ? { mentioned } : {}), ...(pinnedPaths.length > 0 ? { pinnedPaths } : {}), + ...(protocol ? { protocol } : {}), evidenceRefs }; }; @@ -378,6 +384,24 @@ const stringFromAliases = (record: Record, aliases: string[]): return undefined; }; +const protocolSelectionFromRunConfig = ( + runConfig: Record +): EffectiveRunConfig["protocol"] => { + const value = runConfig.protocol; + if (value === undefined) { + return undefined; + } + if (!isRecord(value)) { + throw new Error("INVALID_PROTOCOL_SELECTION"); + } + const protocolId = stringFromAliases(value, ["id", "protocolId", "protocol_id"]); + const protocolVersion = stringFromAliases(value, ["version", "protocolVersion", "protocol_version"]); + if (!protocolId || !protocolVersion) { + throw new Error("INVALID_PROTOCOL_SELECTION"); + } + return { protocolId, protocolVersion }; +}; + const stringArrayFromAliases = (record: Record, aliases: string[]): string[] => { for (const alias of aliases) { const value = record[alias]; diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 4655d5b..395c74b 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -51,6 +51,9 @@ import { sendAuthError } from "./auth/routes.js"; import { createMetadataContextPackageRecorder } from "./context-package-recorder.js"; +import { MetadataProtocolStateStore } from "./protocol-state-store.js"; +import { replayPendingProtocolEvents } from "./protocol-event-recovery.js"; +import { assistantMessageIdFromEvent, completeProtocolRun } from "./protocol-run-completion.js"; import { persistCurrentUserMessage } from "./conversation-memory.js"; import { resolveEvidenceReferenceContext } from "./evidence-reference-context.js"; import { createRunAgentAssembly, createRunAgentContext } from "./run-agent-assembly.js"; @@ -627,6 +630,10 @@ class DataFoundryAgUiAgent extends AbstractAgent { sessionId, userId: this.input.user.id }); + const protocolStateStore = new MetadataProtocolStateStore( + this.input.metadataStore, + this.input.user.id + ); const runAbortController = new AbortController(); const interactionRuntime = new InteractionRuntimeAdapter( this.input.metadataStore, @@ -649,9 +656,17 @@ class DataFoundryAgUiAgent extends AbstractAgent { const emit = (event: BaseEvent): void => { eventPipeline.emit(event); }; + replayPendingProtocolEvents({ runId, stateStore: protocolStateStore, emit }); const agentAssembly = await createRunAgentAssembly({ abortSignal: runAbortController.signal, contextPackageRecorder, + contextPackageExists: (reference) => Boolean( + this.input.metadataStore.contextPackageSnapshots.findByPackageRevision({ + user_id: this.input.user.id, + package_id: reference.packageId, + revision: reference.revision + }) + ), dataGateway: this.input.dataGateway, artifactService: this.input.artifactService, sessionOutputService: this.input.sessionOutputService, @@ -668,6 +683,7 @@ class DataFoundryAgUiAgent extends AbstractAgent { messages: conversationMessages, ...(modelContextProfile ? { modelContextProfile } : {}), modelProvider, + protocolStateStore, ...(modelSettings ? { modelSettings } : {}), runContext, selectedSkills, @@ -701,6 +717,7 @@ class DataFoundryAgUiAgent extends AbstractAgent { let runTimeout: ReturnType | undefined; let terminalStarted = false; let sessionTitleStarted = false; + let lastAssistantMessageId: string | undefined; /** toolCallIds that already persisted TOOL_CALL_START in this run (HITL atomic contract). */ const startedToolCallIds = new Set(); const clearRunTimeout = (): void => { @@ -770,6 +787,10 @@ class DataFoundryAgUiAgent extends AbstractAgent { if (terminalStarted) { return; } + const assistantMessageId = assistantMessageIdFromEvent(event); + if (assistantMessageId) { + lastAssistantMessageId = assistantMessageId; + } const interactionRequested = interactionRuntime.capture(event); if (interactionRequested) { terminalStarted = true; @@ -813,7 +834,24 @@ class DataFoundryAgUiAgent extends AbstractAgent { terminalStarted = true; clearRunTimeout(); unregisterCancel(); - finalization = finalizer.complete({ goalRuntime: agentAssembly.goalRuntime, terminalEvent: event }); + const persistedAssistantMessage = lastAssistantMessageId + ? undefined + : this.input.metadataStore.conversationMessages.findLatestAssistantByRun({ + user_id: this.input.user.id, + session_id: sessionId, + run_id: runId + }); + finalization = completeProtocolRun({ + finalizer, + ...(agentAssembly.goalRuntime ? { goalRuntime: agentAssembly.goalRuntime } : {}), + ...(lastAssistantMessageId ? { lastAssistantMessageId } : {}), + ...(persistedAssistantMessage?.message_id + ? { persistedAssistantMessageId: persistedAssistantMessage.message_id } + : {}), + protocol: agentAssembly.protocol, + runId, + terminalEvent: event + }); return; } if (event.type === EventType.RUN_ERROR) { @@ -829,6 +867,10 @@ class DataFoundryAgUiAgent extends AbstractAgent { } emit(event); + if (event.type === EventType.RUN_STARTED) { + agentAssembly.flushProtocolEvents(); + } + if ( interactionResume && !resumeResolved diff --git a/package-lock.json b/package-lock.json index e3d9f82..f7083c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7569,6 +7569,12 @@ "@types/node": "*" } }, + "node_modules/@types/pegjs": { + "version": "0.10.6", + "resolved": "https://registry.npmjs.org/@types/pegjs/-/pegjs-0.10.6.tgz", + "integrity": "sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==", + "license": "MIT" + }, "node_modules/@types/pg": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", @@ -15760,6 +15766,19 @@ "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", "license": "MIT" }, + "node_modules/node-sql-parser": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/node-sql-parser/-/node-sql-parser-5.4.0.tgz", + "integrity": "sha512-jVe6Z61gPcPjCElPZ6j8llB3wnqGcuQzefim1ERsqIakxnEy5JlzV7XKdO1KmacRG5TKwPc4vJTgSRQ0LfkbFw==", + "license": "Apache-2.0", + "dependencies": { + "@types/pegjs": "^0.10.0", + "big-integer": "^1.6.48" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/nodemailer": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-9.0.3.tgz", @@ -20822,6 +20841,7 @@ "@mastra/core": "^1.46.0", "@mastra/libsql": "^1.14.0", "@mastra/memory": "^1.21.0", + "node-sql-parser": "^5.4.0", "zod": "^4.2.1" } }, diff --git a/package.json b/package.json index 87848a2..feaad08 100644 --- a/package.json +++ b/package.json @@ -86,9 +86,11 @@ "smoke:run-identity": "npm run build && node scripts/smoke-run-identity.mjs", "smoke:run-config-disabled": "npm run build && node scripts/smoke-run-config-disabled.mjs", "smoke:data-gateway": "npm run build && node scripts/smoke-data-gateway.mjs", + "smoke:datalink-semantic": "node scripts/smoke-datalink-semantic.mjs", "smoke:server-datasources": "npm run build && node scripts/smoke-server-datasources-e2e.mjs", "smoke:sql": "npm run build && node scripts/smoke-sql-readonly.mjs", "smoke:agent": "npm run build && node scripts/smoke-agent-runtime.mjs", + "smoke:agent-protocol-deepseek": "npm run build && node scripts/smoke-agent-protocol-deepseek.mjs", "smoke:trace-sections": "npm run build && node scripts/smoke-trace-sections.mjs", "eval:dacomp6": "node scripts/run-dacomp6-complex-case.mjs", "smoke:task-state": "npm run build && node scripts/smoke-task-state.mjs", diff --git a/packages/agent-runtime/package.json b/packages/agent-runtime/package.json index eaa00c8..50dd0da 100644 --- a/packages/agent-runtime/package.json +++ b/packages/agent-runtime/package.json @@ -21,6 +21,7 @@ "@datafoundry/knowledge": "0.2.0", "@datafoundry/providers": "0.2.0", "@datafoundry/skills": "0.2.0", + "node-sql-parser": "^5.4.0", "zod": "^4.2.1" } } diff --git a/packages/agent-runtime/src/capabilities/action-router.test.ts b/packages/agent-runtime/src/capabilities/action-router.test.ts new file mode 100644 index 0000000..eb5ba01 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/action-router.test.ts @@ -0,0 +1,403 @@ +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; + +import { InMemoryProtocolStateStore } from "../protocol/in-memory-protocol-state-store.js"; +import { ProtocolRuntime } from "../protocol/protocol-runtime.js"; +import type { + AgentProtocolDefinition, + ContextPackageRef, + ProtocolEvent, + ProtocolRunState +} from "../protocol/types.js"; +import { ActionRouter } from "./action-router.js"; +import { CapabilityRegistry } from "./capability-registry.js"; + +const contextPackageRef: ContextPackageRef = { packageId: "context-1", revision: 0 }; + +describe("ActionRouter", () => { + it("denies execution when server policy rejects the action", async () => { + let executed = false; + const registry = createRegistry(async () => { + executed = true; + return { value: 1 }; + }); + const runtime = createRuntime(); + const router = new ActionRouter(registry, runtime, { + serverPolicy: () => ({ allowed: false, reasonCode: "SERVER_POLICY_DENIED" }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + + await expect(router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 1 } + })).rejects.toThrow("ACTION_REJECTED:SERVER_POLICY_DENIED:test.calculate"); + expect(executed).toBe(false); + expect(runtime.getState("run-1", "segment-1").actions).toMatchObject([{ + actionName: "test.calculate", + status: "rejected", + reasonCode: "SERVER_POLICY_DENIED" + }]); + }); + + it("validates, executes, reduces state, and records the output context", async () => { + const registry = createRegistry(async () => ({ value: 2 })); + const runtime = createRuntime(); + const router = new ActionRouter(registry, runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + + const result = await router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 } + }); + + expect(result).toMatchObject({ + rawResult: { value: 2 }, + observation: { value: 2 }, + contextPackageRef: { packageId: "context-1", revision: 1 } + }); + expect(runtime.getState("run-1", "segment-1")).toMatchObject({ + revision: 2, + domain: { total: 2 } + }); + }); + + it("returns the recorded result for a repeated idempotency key", async () => { + let executions = 0; + const registry = createRegistry(async () => { + executions += 1; + return { value: 2 }; + }); + const runtime = createRuntime(); + const router = new ActionRouter(registry, runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + const input = { + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 }, + idempotencyKey: "same-calculation" + }; + + const first = await router.execute(input); + const second = await router.execute({ ...input, actionId: "action-2" }); + + expect(second).toEqual(first); + expect(executions).toBe(1); + expect(runtime.getState("run-1", "segment-1").actions).toHaveLength(1); + }); + + it("returns the governed context package and observation projection", async () => { + const router = new ActionRouter(createRegistry(async () => ({ value: 2 })), createRuntime(), { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ + contextPackageRef: { packageId: "context-1", revision: 1 }, + contextPackage: { packageId: "context-1", revision: 1, items: [] }, + observation: { summary: "calculated" } + }) + }); + + const result = await router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 } + }); + + expect(result.observation).toEqual({ summary: "calculated" }); + expect(result.contextPackage).toMatchObject({ packageId: "context-1", revision: 1 }); + }); + + it("executes runtime automatic actions after a successful primary action", async () => { + let revision = 0; + const runtime = createRuntime(); + const router = new ActionRouter(createRegistry(async () => ({ value: 2 })), runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: ++revision }), + automaticActions: ({ actionName }) => actionName === "test.calculate" + ? [{ actionName: "test.audit", input: { valid: true } }] + : [] + }); + + await router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 } + }); + + expect(runtime.getState("run-1", "segment-1").actions.map((action) => action.actionName)) + .toEqual(["test.calculate", "test.audit"]); + }); + + it("projects the model-visible observation after automatic actions update protocol state", async () => { + let revision = 0; + const runtime = createRuntime(); + const router = new ActionRouter(createRegistry(async () => ({ value: 2 })), runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: ++revision }), + automaticActions: ({ actionName }) => actionName === "test.calculate" + ? [{ actionName: "test.audit", input: { valid: true } }] + : [], + projectFinalObservation: ({ actionName, domain, observation }) => ({ + ...(observation as Record), + actionName, + protocolTotal: (domain as { total: number }).total + }) + }); + + const result = await router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 } + }); + + expect(result.observation).toEqual({ value: 2, actionName: "test.calculate", protocolTotal: 2 }); + expect(runtime.getState("run-1", "segment-1").actions.map((action) => action.actionName)) + .toEqual(["test.calculate", "test.audit"]); + }); + + it("executes runtime preparatory actions before checking the primary action phase", async () => { + const registry = createRegistry(async () => ({ value: 2 })); + const definition: AgentProtocolDefinition<{ total: number }> = { + ...createProtocol(), + initialPhase: "prepare", + phases: { + prepare: { + allowedActions: ["test.audit"], + transitions: [{ targetPhase: "active", when: ({ actionName }) => actionName === "test.audit" }] + }, + active: { allowedActions: ["test.calculate"], transitions: [] } + } + }; + const runtime = new ProtocolRuntime(definition, new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + let revision = 0; + const router = new ActionRouter(registry, runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: ++revision }), + preparatoryActions: ({ actionName }) => actionName === "test.calculate" + ? [{ actionName: "test.audit", input: { valid: true } }] + : [] + }); + + await router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 } + }); + + expect(runtime.getState("run-1", "segment-1").actions.map((action) => action.actionName)) + .toEqual(["test.audit", "test.calculate"]); + }); + + it("records a failed action when its executor throws", async () => { + const runtime = createRuntime(); + const router = new ActionRouter(createRegistry(async () => { + throw new Error("EXECUTOR_FAILED"); + }), runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + + await expect(router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-failed", + actionName: "test.calculate", + input: { value: 2 } + })).rejects.toThrow("EXECUTOR_FAILED"); + + expect(runtime.getState("run-1", "segment-1").actions).toMatchObject([{ + actionId: "action-failed", + actionName: "test.calculate", + status: "failed", + reasonCode: "EXECUTOR_FAILED" + }]); + }); + + it("reports succeeded_uncommitted when external execution returns an invalid result", async () => { + const runtime = createRuntime(); + const router = new ActionRouter(createRegistry(async () => ({ value: "invalid" })), runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + + await expect(router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-invalid-result", + actionName: "test.calculate", + input: { value: 2 } + })).rejects.toMatchObject({ + observation: { + error: { executionStatus: "succeeded_uncommitted" }, + recovery: { strategy: "refresh_and_replan" } + } + }); + expect(runtime.getState("run-1", "segment-1").actions).toMatchObject([{ + actionId: "action-invalid-result", + status: "failed" + }]); + }); + + it("commits parallel action results without re-executing tools or losing domain updates", async () => { + const releases = new Map void>(); + const executions: number[] = []; + const registry = createRegistry(async (input) => { + const value = (input as { value: number }).value; + executions.push(value); + await new Promise((resolve) => releases.set(value, resolve)); + return { value }; + }); + const runtime = createRuntime(); + let contextRevision = 0; + const router = new ActionRouter(registry, runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: ++contextRevision }) + }); + + const first = router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 1 } + }); + const second = router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-2", + actionName: "test.calculate", + input: { value: 2 } + }); + await vi.waitFor(() => expect(releases.size).toBe(2)); + + releases.get(2)?.(); + await second; + releases.get(1)?.(); + await first; + + expect(executions.sort()).toEqual([1, 2]); + expect(runtime.getState("run-1", "segment-1")).toMatchObject({ + revision: 4, + domain: { total: 3 }, + actions: [ + { actionId: "action-1", status: "succeeded" }, + { actionId: "action-2", status: "succeeded" } + ] + }); + }); + + it("replays the reducer on the latest domain after a compare-and-set conflict", async () => { + let executions = 0; + const store = new ConflictOnceProtocolStateStore(); + const runtime = new ProtocolRuntime(createProtocol(), store); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + const router = new ActionRouter(createRegistry(async () => { + executions += 1; + return { value: 2 }; + }), runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + store.injectConflictOnNextCommit(); + + await router.execute({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "test.calculate", + input: { value: 2 } + }); + + expect(executions).toBe(1); + expect(runtime.getState("run-1", "segment-1")).toMatchObject({ + revision: 3, + domain: { total: 12 }, + actions: [{ actionId: "action-1", status: "succeeded" }] + }); + }); +}); + +class ConflictOnceProtocolStateStore extends InMemoryProtocolStateStore { + private shouldInjectConflict = false; + + injectConflictOnNextCommit(): void { + this.shouldInjectConflict = true; + } + + override compareAndSet( + state: ProtocolRunState, + expectedRevision: number, + events: ProtocolEvent[] = [] + ): ProtocolRunState { + if (this.shouldInjectConflict && state.actions.some((action) => action.status === "succeeded")) { + this.shouldInjectConflict = false; + const current = this.get<{ total: number }>(state.runId, state.segmentId); + super.compareAndSet({ + ...current, + revision: current.revision + 1, + domain: { total: current.domain.total + 10 } + }, expectedRevision); + } + return super.compareAndSet(state, expectedRevision, events); + } +} + +const createRuntime = (): ProtocolRuntime<{ total: number }> => { + const runtime = new ProtocolRuntime(createProtocol(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + return runtime; +}; + +const createProtocol = (): AgentProtocolDefinition<{ total: number }> => ({ + id: "test/protocol", + version: "1", + initialPhase: "active", + phases: { active: { allowedActions: ["test.calculate", "test.audit"], transitions: [] } }, + createInitialState: () => ({ total: 0 }), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) +}); + +const createRegistry = (execute: (input: unknown) => Promise): CapabilityRegistry => { + const registry = new CapabilityRegistry(); + registry.register({ + manifest: { id: "test-plugin", version: "1", provides: ["test.calculate", "test.audit"] }, + actions: [{ + name: "test.calculate", + exposure: "agent", + inputSchema: z.object({ value: z.number() }), + outputSchema: z.object({ value: z.number() }), + idempotency: "supported", + execute: async (_context, input) => execute(input), + reduce: (domain, result) => ({ + total: (domain as { total: number }).total + (result as { value: number }).value + }) + }, { + name: "test.audit", + exposure: "runtime", + inputSchema: z.object({ valid: z.boolean() }), + outputSchema: z.object({ valid: z.boolean() }), + idempotency: "supported", + execute: async (_context, input) => input + }] + }); + return registry; +}; diff --git a/packages/agent-runtime/src/capabilities/action-router.ts b/packages/agent-runtime/src/capabilities/action-router.ts new file mode 100644 index 0000000..924fed8 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/action-router.ts @@ -0,0 +1,312 @@ +import { toToolExecutionError } from "../errors/tool-execution-error.js"; +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; +import type { ContextPackageRef, ProtocolGuardResult } from "../protocol/types.js"; +import type { ProtocolRuntime } from "../protocol/protocol-runtime.js"; +import type { CapabilityRegistry } from "./capability-registry.js"; + +export type ActionRouterOptions = { + afterPreparatoryActions?(input: { + actionName: string; + domain: unknown; + input: unknown; + phase: string; + }): Promise | void; + afterAction?(input: { + actionName: string; + rawResult: unknown; + }): Promise | void; + preparatoryActions?(input: { + actionName: string; + input: unknown; + }): Array<{ actionName: string; input: unknown }>; + automaticActions?(input: { + actionName: string; + domain: unknown; + input: unknown; + observation: unknown; + rawResult: unknown; + }): Array<{ actionName: string; input: unknown }>; + serverPolicy(input: { actionName: string; input: unknown }): ProtocolGuardResult; + resourceAuthorization?(input: { actionName: string; input: unknown }): ProtocolGuardResult; + projectContext(input: { + actionName: string; + observation: unknown; + rawResult: unknown; + }): ContextPackageRef | ActionContextProjection; + projectProtocolEventResult?(input: { + actionName: string; + rawResult: unknown; + }): unknown; + projectFinalObservation?(input: { + actionName: string; + domain: unknown; + observation: unknown; + rawResult: unknown; + }): unknown; +}; + +export type ActionContextProjection = { + contextPackageRef: ContextPackageRef; + contextPackage?: unknown; + observation?: unknown; +}; + +export type ExecuteActionInput = { + runId: string; + segmentId: string; + actionId: string; + actionName: string; + input: unknown; + idempotencyKey?: string; + abortSignal?: AbortSignal; + invocationArgs?: unknown[]; + automaticDepth?: number; +}; + +export type ActionExecutionResult = { + rawResult: unknown; + observation: unknown; + contextPackageRef: ContextPackageRef; + contextPackage?: unknown; +}; + +export class ActionRouter { + private readonly idempotentResults = new Map(); + + constructor( + private readonly registry: CapabilityRegistry, + private protocolRuntime: ProtocolRuntime, + private readonly options: ActionRouterOptions + ) {} + + async execute(input: ExecuteActionInput): Promise { + const policy = this.options.serverPolicy({ actionName: input.actionName, input: input.input }); + if (!policy.allowed) { + this.rejectAction(input, policy.reasonCode, `ACTION_REJECTED:${policy.reasonCode}:${input.actionName}`); + } + const authorization = this.options.resourceAuthorization?.({ + actionName: input.actionName, + input: input.input + }); + if (authorization && !authorization.allowed) { + this.rejectAction( + input, + authorization.reasonCode, + `ACTION_REJECTED:${authorization.reasonCode}:${input.actionName}` + ); + } + const automaticDepth = input.automaticDepth ?? 0; + const preparatoryActions = this.options.preparatoryActions?.({ + actionName: input.actionName, + input: input.input + }) ?? []; + if (preparatoryActions.length > 0 && automaticDepth >= AGENT_RUNTIME_LIMITS.protocolAutomaticActionMaxDepth) { + throw new Error("PROTOCOL_AUTOMATIC_ACTION_DEPTH_EXCEEDED"); + } + for (const [index, preparatoryAction] of preparatoryActions.entries()) { + await this.execute({ + runId: input.runId, + segmentId: input.segmentId, + actionId: `${input.actionId}:prepare:${index + 1}`, + actionName: preparatoryAction.actionName, + input: preparatoryAction.input, + automaticDepth: automaticDepth + 1, + idempotencyKey: `${input.actionId}:prepare:${index + 1}`, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}) + }); + } + const preparedState = this.protocolRuntime.getState(input.runId, input.segmentId); + await this.options.afterPreparatoryActions?.({ + actionName: input.actionName, + domain: preparedState.domain, + input: input.input, + phase: preparedState.phase + }); + const registered = this.registry.resolve(input.actionName); + if (!registered) { + throw new Error(`CAPABILITY_ACTION_NOT_REGISTERED:${input.actionName}`); + } + const parsedInput = registered.action.inputSchema.parse(input.input); + const pluginGuard = await registered.action.guard?.(parsedInput); + if (pluginGuard && !pluginGuard.allowed) { + throw new Error(`ACTION_REJECTED:${pluginGuard.reasonCode}:${input.actionName}`); + } + if (registered.action.idempotency === "required" && !input.idempotencyKey) { + throw new Error(`ACTION_IDEMPOTENCY_KEY_REQUIRED:${input.actionName}`); + } + const idempotencyKey = input.idempotencyKey && registered.action.idempotency !== "none" + ? `${input.runId}:${input.segmentId}:${input.actionName}:${input.idempotencyKey}` + : undefined; + const existing = idempotencyKey ? this.idempotentResults.get(idempotencyKey) : undefined; + if (existing) { + return existing; + } + try { + this.protocolRuntime.beginAction({ + runId: input.runId, + segmentId: input.segmentId, + actionId: input.actionId, + actionName: input.actionName, + actionInput: parsedInput + }); + } catch (error) { + const message = error instanceof Error ? error.message : "ACTION_NOT_ALLOWED"; + if (isAdmissionRejection(message)) { + this.rejectAction(input, message.split(":", 1)[0] ?? "ACTION_NOT_ALLOWED", message); + } + throw error; + } + let executionResult: unknown; + let rawResult: unknown; + let observation: unknown; + let contextPackageRef: ContextPackageRef; + let contextPackage: unknown; + try { + executionResult = await registered.action.execute({ + actionId: input.actionId, + actionName: input.actionName, + runId: input.runId, + segmentId: input.segmentId, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}), + ...(input.invocationArgs ? { invocationArgs: input.invocationArgs } : {}) + }, parsedInput); + } catch (error) { + try { + this.protocolRuntime.recordActionFailure({ + runId: input.runId, + segmentId: input.segmentId, + actionId: input.actionId, + actionName: input.actionName, + reasonCode: error instanceof Error ? error.message : "ACTION_EXECUTION_FAILED" + }); + } catch { + // Preserve the executor failure: state contention must not replace the actionable root cause. + } + throw toToolExecutionError(error, { + executionStatus: "failed", + idempotency: registered.action.idempotency, + toolName: input.actionName + }); + } + try { + rawResult = registered.action.outputSchema.parse(executionResult); + const initialObservation = registered.action.projectObservation?.(rawResult) ?? rawResult; + const projection = this.options.projectContext({ + actionName: input.actionName, + observation: initialObservation, + rawResult + }); + const normalizedProjection = isContextPackageRef(projection) + ? { contextPackageRef: projection } + : projection; + observation = normalizedProjection.observation ?? initialObservation; + contextPackageRef = normalizedProjection.contextPackageRef; + contextPackage = normalizedProjection.contextPackage; + this.protocolRuntime.recordActionSuccess({ + runId: input.runId, + segmentId: input.segmentId, + actionId: input.actionId, + actionName: input.actionName, + ...(registered.action.reduce + ? { reduceDomain: (domain: TDomainState) => registered.action.reduce?.(domain, rawResult) as TDomainState } + : {}), + ...(this.options.projectProtocolEventResult + ? { eventResult: this.options.projectProtocolEventResult({ actionName: input.actionName, rawResult }) } + : {}), + outputContextPackageRef: contextPackageRef + }); + } catch (error) { + if (!isProtocolCommitContention(error)) { + try { + this.protocolRuntime.recordActionFailure({ + runId: input.runId, + segmentId: input.segmentId, + actionId: input.actionId, + actionName: input.actionName, + reasonCode: error instanceof Error ? error.message : "ACTION_RESULT_PROCESSING_FAILED" + }); + } catch { + // The model-facing error below still preserves the original post-execution failure. + } + } + throw toToolExecutionError(error, { + executionStatus: "succeeded_uncommitted", + idempotency: registered.action.idempotency, + toolName: input.actionName + }, { + rawResult: executionResult + }); + } + let result: ActionExecutionResult = { + rawResult, + observation, + contextPackageRef, + ...(contextPackage === undefined ? {} : { contextPackage }) + }; + await this.options.afterAction?.({ actionName: input.actionName, rawResult }); + const automaticActions = this.options.automaticActions?.({ + actionName: input.actionName, + domain: this.protocolRuntime.getState(input.runId, input.segmentId).domain, + input: input.input, + observation, + rawResult + }) ?? []; + if (automaticActions.length > 0 && automaticDepth >= 10) { + throw new Error("PROTOCOL_AUTOMATIC_ACTION_DEPTH_EXCEEDED"); + } + for (const [index, automaticAction] of automaticActions.entries()) { + await this.execute({ + runId: input.runId, + segmentId: input.segmentId, + actionId: `${input.actionId}:auto:${index + 1}`, + actionName: automaticAction.actionName, + input: automaticAction.input, + automaticDepth: automaticDepth + 1, + idempotencyKey: `${input.actionId}:auto:${index + 1}`, + ...(input.abortSignal ? { abortSignal: input.abortSignal } : {}) + }); + } + if (this.options.projectFinalObservation) { + result = { + ...result, + observation: this.options.projectFinalObservation({ + actionName: input.actionName, + domain: this.protocolRuntime.getState(input.runId, input.segmentId).domain, + observation: result.observation, + rawResult: result.rawResult + }) + }; + } + if (idempotencyKey) { + this.idempotentResults.set(idempotencyKey, result); + } + return result; + } + + replaceProtocolRuntime(runtime: ProtocolRuntime): void { + this.protocolRuntime = runtime; + } + + private rejectAction(input: ExecuteActionInput, reasonCode: string, message: string): never { + this.protocolRuntime.recordActionRejection({ + runId: input.runId, + segmentId: input.segmentId, + actionId: input.actionId, + actionName: input.actionName, + reasonCode + }); + throw new Error(message); + } +} + +const isContextPackageRef = ( + value: ContextPackageRef | ActionContextProjection +): value is ContextPackageRef => "packageId" in value && "revision" in value; + +const isAdmissionRejection = (message: string): boolean => + message.startsWith("ACTION_NOT_ALLOWED_IN_PHASE:") + || message.startsWith("PROTOCOL_ACTION_BUDGET_EXHAUSTED:") + || message.startsWith("PROTOCOL_GUARD_REJECTED:"); + +const isProtocolCommitContention = (error: unknown): boolean => + error instanceof Error && error.message.startsWith("PROTOCOL_COMMIT_CONTENTION:"); diff --git a/packages/agent-runtime/src/capabilities/capability-registry.test.ts b/packages/agent-runtime/src/capabilities/capability-registry.test.ts new file mode 100644 index 0000000..47d3eb0 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/capability-registry.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { CapabilityRegistry } from "./capability-registry.js"; +import type { CapabilityPlugin } from "./types.js"; + +describe("CapabilityRegistry", () => { + it("rejects action namespace conflicts across plugins", () => { + const registry = new CapabilityRegistry(); + registry.register(createPlugin("plugin-a", "data.schema.inspect")); + + expect(() => registry.register(createPlugin("plugin-b", "data.schema.inspect"))) + .toThrow("CAPABILITY_ACTION_ALREADY_REGISTERED:data.schema.inspect:plugin-a:plugin-b"); + }); + + it("rejects a plugin whose required dependency is missing", () => { + const registry = new CapabilityRegistry(); + const plugin = createPlugin("plugin-b", "data.query.execute"); + plugin.manifest.requires = [{ id: "plugin-a", version: "1" }]; + + expect(() => registry.register(plugin)) + .toThrow("CAPABILITY_PLUGIN_DEPENDENCY_MISSING:plugin-b:plugin-a@1"); + }); + + it("initializes plugins in registration order and disposes them in reverse order", async () => { + const lifecycle: string[] = []; + const registry = new CapabilityRegistry(); + const first = createPlugin("plugin-a", "data.schema.inspect"); + first.initialize = async () => { lifecycle.push("init:a"); }; + first.dispose = async () => { lifecycle.push("dispose:a"); }; + const second = createPlugin("plugin-b", "data.query.execute"); + second.initialize = async () => { lifecycle.push("init:b"); }; + second.dispose = async () => { lifecycle.push("dispose:b"); }; + registry.register(first); + registry.register(second); + + await registry.initialize(); + await registry.dispose(); + + expect(lifecycle).toEqual(["init:a", "init:b", "dispose:b", "dispose:a"]); + }); +}); + +const createPlugin = (pluginId: string, actionName: string): CapabilityPlugin => ({ + manifest: { id: pluginId, version: "1", provides: [actionName] }, + actions: [{ + name: actionName, + exposure: "agent", + inputSchema: z.object({}), + outputSchema: z.object({ ok: z.boolean() }), + idempotency: "supported", + execute: async () => ({ ok: true }) + }] +}); diff --git a/packages/agent-runtime/src/capabilities/capability-registry.ts b/packages/agent-runtime/src/capabilities/capability-registry.ts new file mode 100644 index 0000000..0487633 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/capability-registry.ts @@ -0,0 +1,80 @@ +import type { + CapabilityActionDefinition, + CapabilityExposure, + CapabilityPlugin, + RegisteredCapabilityAction +} from "./types.js"; + +export class CapabilityRegistry { + private readonly actions = new Map(); + private readonly plugins = new Map(); + private initializedPlugins: CapabilityPlugin[] = []; + + register(plugin: CapabilityPlugin): void { + for (const dependency of plugin.manifest.requires ?? []) { + const registered = this.plugins.get(dependency.id); + if (!registered || (dependency.version && registered.manifest.version !== dependency.version)) { + throw new Error( + `CAPABILITY_PLUGIN_DEPENDENCY_MISSING:${plugin.manifest.id}:${dependency.id}@` + + (dependency.version ?? "any") + ); + } + } + for (const action of plugin.actions) { + const existing = this.actions.get(action.name); + if (existing) { + throw new Error( + `CAPABILITY_ACTION_ALREADY_REGISTERED:${action.name}:${existing.pluginId}:${plugin.manifest.id}` + ); + } + } + this.plugins.set(plugin.manifest.id, plugin); + for (const action of plugin.actions) { + this.actions.set(action.name, { + action, + pluginId: plugin.manifest.id, + pluginVersion: plugin.manifest.version + }); + } + } + + resolve(actionName: string): RegisteredCapabilityAction | undefined { + return this.actions.get(actionName); + } + + listByExposure(exposure: CapabilityExposure): CapabilityActionDefinition[] { + return [...this.actions.values()] + .map((entry) => entry.action) + .filter((action) => action.exposure === exposure || action.exposure === "both"); + } + + async initialize(): Promise { + if (this.initializedPlugins.length > 0) { + return; + } + try { + for (const plugin of this.plugins.values()) { + await plugin.initialize?.(); + this.initializedPlugins.push(plugin); + } + } catch (error) { + await this.dispose(); + throw error; + } + } + + async dispose(): Promise { + const failures: unknown[] = []; + for (const plugin of [...this.initializedPlugins].reverse()) { + try { + await plugin.dispose?.(); + } catch (error) { + failures.push(error); + } + } + this.initializedPlugins = []; + if (failures.length > 0) { + throw new AggregateError(failures, "CAPABILITY_PLUGIN_DISPOSE_FAILED"); + } + } +} diff --git a/packages/agent-runtime/src/capabilities/tool-capability-plugin.test.ts b/packages/agent-runtime/src/capabilities/tool-capability-plugin.test.ts new file mode 100644 index 0000000..0fd7907 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/tool-capability-plugin.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; + +import { createToolCapabilityPlugin } from "./tool-capability-plugin.js"; + +describe("createToolCapabilityPlugin", () => { + it("preserves the original tool invocation arguments", async () => { + const calls: unknown[][] = []; + const plugin = createToolCapabilityPlugin({ + id: "existing-tools", + tools: { + test_tool: { + execute: async (...args: unknown[]) => { + calls.push(args); + return { ok: true }; + } + } + } + }); + const action = plugin.actions[0]; + if (!action) { + throw new Error("TEST_ACTION_REQUIRED"); + } + + const result = await action.execute({ + actionId: "action-1", + actionName: "test_tool", + runId: "run-1", + segmentId: "segment-1", + invocationArgs: [{ toolCallId: "call-1" }] + }, { value: 1 }); + + expect(result).toEqual({ ok: true }); + expect(calls).toEqual([[{ value: 1 }, { toolCallId: "call-1" }]]); + }); + + it("binds each existing tool to the selected protocol reducer", () => { + const plugin = createToolCapabilityPlugin({ + id: "existing-tools", + tools: { test_tool: { execute: async () => ({ value: 2 }) } }, + reduceAction: (state, actionName, result) => ({ + total: (state as { total: number }).total + (result as { value: number }).value, + actionName + }) + }); + const action = plugin.actions[0]; + if (!action?.reduce) { + throw new Error("TEST_REDUCER_REQUIRED"); + } + + expect(action.reduce({ total: 1 }, { value: 2 })).toEqual({ total: 3, actionName: "test_tool" }); + }); +}); diff --git a/packages/agent-runtime/src/capabilities/tool-capability-plugin.ts b/packages/agent-runtime/src/capabilities/tool-capability-plugin.ts new file mode 100644 index 0000000..3c590a8 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/tool-capability-plugin.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; + +import type { CapabilityPlugin } from "./types.js"; + +type ExecutableTool = { + execute?: (...args: unknown[]) => unknown | Promise; +}; + +export type CreateToolCapabilityPluginInput = { + id: string; + version?: string; + tools: Record; + reduceAction?(domainState: unknown, actionName: string, result: unknown): unknown; +}; + +/** Adapt an existing tool record into a statically registered capability plugin. */ +export const createToolCapabilityPlugin = (input: CreateToolCapabilityPluginInput): CapabilityPlugin => ({ + manifest: { + id: input.id, + version: input.version ?? "1", + provides: Object.keys(input.tools) + }, + actions: Object.entries(input.tools).map(([name, tool]) => { + if (!tool.execute) { + throw new Error(`TOOL_EXECUTE_REQUIRED:${name}`); + } + const execute = tool.execute; + return { + name, + exposure: "agent" as const, + inputSchema: z.unknown(), + outputSchema: z.unknown(), + idempotency: "none" as const, + execute: async (context, actionInput) => execute(actionInput, ...(context.invocationArgs ?? [])), + ...(input.reduceAction + ? { reduce: (domainState: unknown, result: unknown) => input.reduceAction?.(domainState, name, result) } + : {}) + }; + }) +}); diff --git a/packages/agent-runtime/src/capabilities/types.ts b/packages/agent-runtime/src/capabilities/types.ts new file mode 100644 index 0000000..edc1e10 --- /dev/null +++ b/packages/agent-runtime/src/capabilities/types.ts @@ -0,0 +1,48 @@ +import type { z } from "zod"; + +export type CapabilityExposure = "agent" | "runtime" | "both"; + +export type CapabilityPluginManifest = { + id: string; + version: string; + provides: string[]; + requires?: Array<{ id: string; version?: string }>; +}; + +export type CapabilityActionGuardResult = + | { allowed: true } + | { allowed: false; reasonCode: string; message?: string }; + +export type CapabilityActionDefinition = { + name: string; + exposure: CapabilityExposure; + inputSchema: z.ZodType; + outputSchema: z.ZodType; + idempotency: "required" | "supported" | "none"; + guard?(input: unknown): CapabilityActionGuardResult | Promise; + execute(context: CapabilityExecutionContext, input: unknown): Promise; + reduce?(domainState: unknown, result: unknown): unknown; + projectObservation?(result: unknown): unknown; +}; + +export type CapabilityExecutionContext = { + actionId: string; + actionName: string; + runId: string; + segmentId: string; + abortSignal?: AbortSignal; + invocationArgs?: unknown[]; +}; + +export type CapabilityPlugin = { + manifest: CapabilityPluginManifest; + actions: CapabilityActionDefinition[]; + initialize?(): Promise; + dispose?(): Promise; +}; + +export type RegisteredCapabilityAction = { + action: CapabilityActionDefinition; + pluginId: string; + pluginVersion: string; +}; diff --git a/packages/agent-runtime/src/config/agent-runtime-limits.ts b/packages/agent-runtime/src/config/agent-runtime-limits.ts new file mode 100644 index 0000000..5f0759e --- /dev/null +++ b/packages/agent-runtime/src/config/agent-runtime-limits.ts @@ -0,0 +1,234 @@ +export type AgentRuntimeLimitDefinition = { + defaultValue: number; + min: number; + max: number; + env: string; + description: string; +}; + +/** + * Registry for every configurable numeric limit owned by agent-runtime. + * + * Defaults, bounds, environment names, and operational intent live together so callers never introduce + * unexplained numeric thresholds in workflow code. + */ +export const AGENT_RUNTIME_LIMIT_DEFINITIONS = { + agentMaxSteps: { + defaultValue: 80, + min: 10, + max: 500, + env: "DATAFOUNDRY_AGENT_MAX_STEPS", + description: "Maximum model reasoning and tool-use steps available to one Agent run." + }, + sqlMaxExecutionCount: { + defaultValue: 60, + min: 1, + max: 500, + env: "DATAFOUNDRY_SQL_MAX_EXECUTION_COUNT", + description: "Maximum uncached read-only SQL executions allowed during one Agent run." + }, + dataAnalysisMaxProtocolActions: { + defaultValue: 500, + min: 50, + max: 5000, + env: "DATAFOUNDRY_DATA_ANALYSIS_MAX_PROTOCOL_ACTIONS", + description: "Maximum governed protocol actions retained for one data-analysis segment." + }, + generalTaskMaxProtocolActions: { + defaultValue: 100, + min: 10, + max: 1000, + env: "DATAFOUNDRY_GENERAL_TASK_MAX_PROTOCOL_ACTIONS", + description: "Maximum governed protocol actions retained for one general-task segment." + }, + protocolDefaultMaxActions: { + defaultValue: 100, + min: 10, + max: 5000, + env: "DATAFOUNDRY_PROTOCOL_DEFAULT_MAX_ACTIONS", + description: "Fallback action budget used when a ProtocolRuntime caller provides no explicit budget." + }, + protocolMaxCompletionRejections: { + defaultValue: 3, + min: 1, + max: 100, + env: "DATAFOUNDRY_PROTOCOL_MAX_COMPLETION_REJECTIONS", + description: "Maximum rejected completion proposals before a run is finalized as partial." + }, + protocolMaxCommitRetries: { + defaultValue: 8, + min: 0, + max: 100, + env: "DATAFOUNDRY_PROTOCOL_MAX_COMMIT_RETRIES", + description: "Maximum optimistic protocol state commit retries after concurrent revision changes." + }, + protocolAutomaticActionMaxDepth: { + defaultValue: 10, + min: 1, + max: 100, + env: "DATAFOUNDRY_PROTOCOL_AUTOMATIC_ACTION_MAX_DEPTH", + description: "Maximum nesting depth for preparatory and automatic protocol actions." + }, + schemaMaxTables: { + defaultValue: 20, + min: 1, + max: 500, + env: "DATAFOUNDRY_SCHEMA_MAX_TABLES", + description: "Maximum inspected tables materialized into model-visible schema context." + }, + schemaMaxColumnsPerTable: { + defaultValue: 50, + min: 1, + max: 1000, + env: "DATAFOUNDRY_SCHEMA_MAX_COLUMNS_PER_TABLE", + description: "Maximum columns per inspected table materialized into model-visible schema context." + }, + sqlMaxModelRows: { + defaultValue: 20, + min: 1, + max: 1000, + env: "DATAFOUNDRY_SQL_MAX_MODEL_ROWS", + description: "Maximum SQL result rows included in the compact observation shown to the model." + }, + sqlMaxActivityRows: { + defaultValue: 20, + min: 1, + max: 1000, + env: "DATAFOUNDRY_SQL_MAX_ACTIVITY_ROWS", + description: "Maximum SQL result rows included in activity and trace event previews." + }, + sqlMaxCellChars: { + defaultValue: 500, + min: 32, + max: 100000, + env: "DATAFOUNDRY_SQL_MAX_CELL_CHARS", + description: "Maximum characters retained from one SQL result cell in model-visible context." + }, + sqlMaxSqlChars: { + defaultValue: 4000, + min: 256, + max: 100000, + env: "DATAFOUNDRY_SQL_MAX_SQL_CHARS", + description: "Maximum SQL text characters retained in activity, trace, and context previews." + }, + contextMaxTokens: { + defaultValue: 32000, + min: 1000, + max: 1000000, + env: "DATAFOUNDRY_CONTEXT_MAX_TOKENS", + description: "Maximum estimated tokens admitted into one assembled Agent context package." + }, + contextMaxChars: { + defaultValue: 32000, + min: 1000, + max: 4000000, + env: "DATAFOUNDRY_CONTEXT_MAX_CHARS", + description: "Maximum source characters admitted before token-aware Agent context shaping." + }, + toolErrorMaxMessageChars: { + defaultValue: 500, + min: 100, + max: 10000, + env: "DATAFOUNDRY_TOOL_ERROR_MAX_MESSAGE_CHARS", + description: "Maximum sanitized tool-error message characters returned to the model." + }, + knowledgeMaxTopK: { + defaultValue: 20, + min: 1, + max: 100, + env: "DATAFOUNDRY_KNOWLEDGE_MAX_TOP_K", + description: "Maximum knowledge retrieval result count accepted from one Agent tool call." + }, + requirementCommitMaxClaims: { + defaultValue: 16, + min: 1, + max: 100, + env: "DATAFOUNDRY_REQUIREMENT_COMMIT_MAX_CLAIMS", + description: "Maximum requirement claims accepted in one analysis commit tool call." + }, + requirementCommitMaxOutputFields: { + defaultValue: 32, + min: 1, + max: 500, + env: "DATAFOUNDRY_REQUIREMENT_COMMIT_MAX_OUTPUT_FIELDS", + description: "Maximum named result fields attached to one committed analysis claim." + }, + modelHelperMaxSteps: { + defaultValue: 1, + min: 1, + max: 10, + env: "DATAFOUNDRY_MODEL_HELPER_MAX_STEPS", + description: "Maximum model steps used by deterministic classifier, extraction, and grounding helpers." + }, + protocolClassifierMaxOutputTokens: { + defaultValue: 512, + min: 64, + max: 4096, + env: "DATAFOUNDRY_PROTOCOL_CLASSIFIER_MAX_OUTPUT_TOKENS", + description: "Maximum output tokens for protocol classification." + }, + requirementExtractorMaxOutputTokens: { + defaultValue: 4096, + min: 256, + max: 32768, + env: "DATAFOUNDRY_REQUIREMENT_EXTRACTOR_MAX_OUTPUT_TOKENS", + description: "Maximum output tokens for logical analysis requirement extraction." + }, + contractGrounderMaxOutputTokens: { + defaultValue: 8192, + min: 512, + max: 65536, + env: "DATAFOUNDRY_CONTRACT_GROUNDER_MAX_OUTPUT_TOKENS", + description: "Maximum output tokens for schema-grounded analysis contract generation." + }, + contractGrounderMaxAttempts: { + defaultValue: 2, + min: 1, + max: 5, + env: "DATAFOUNDRY_CONTRACT_GROUNDER_MAX_ATTEMPTS", + description: "Maximum model attempts for producing one schema-valid grounded analysis contract." + }, + toolObservationMaxNames: { + defaultValue: 5, + min: 1, + max: 100, + env: "DATAFOUNDRY_TOOL_OBSERVATION_MAX_NAMES", + description: "Maximum table or column names retained in a compact tool observation summary." + }, + toolObservationMaxNameChars: { + defaultValue: 120, + min: 16, + max: 2000, + env: "DATAFOUNDRY_TOOL_OBSERVATION_MAX_NAME_CHARS", + description: "Maximum characters retained for one name in a compact tool observation summary." + }, + toolObservationMaxChars: { + defaultValue: 12000, + min: 1000, + max: 100000, + env: "DATAFOUNDRY_TOOL_OBSERVATION_MAX_CHARS", + description: "Maximum characters retained for a default compact tool observation." + } +} as const satisfies Record; + +type ResolvedAgentRuntimeLimits = { + [Key in keyof typeof AGENT_RUNTIME_LIMIT_DEFINITIONS]: number; +}; + +/** Resolve all Agent runtime limits once from documented defaults and optional environment overrides. */ +export const AGENT_RUNTIME_LIMITS: Readonly = Object.freeze( + Object.fromEntries(Object.entries(AGENT_RUNTIME_LIMIT_DEFINITIONS).map(([key, definition]) => [ + key, + readBoundedInteger(process.env[definition.env], definition) + ])) as ResolvedAgentRuntimeLimits +); + +function readBoundedInteger(rawValue: string | undefined, definition: AgentRuntimeLimitDefinition): number { + if (rawValue === undefined || rawValue.trim() === "") { + return definition.defaultValue; + } + const parsed = Number.parseInt(rawValue, 10); + return Number.isSafeInteger(parsed) && parsed >= definition.min && parsed <= definition.max + ? parsed + : definition.defaultValue; +} diff --git a/packages/agent-runtime/src/context/inventory/context-limits.ts b/packages/agent-runtime/src/context/inventory/context-limits.ts index eeecd76..ba4c0a3 100644 --- a/packages/agent-runtime/src/context/inventory/context-limits.ts +++ b/packages/agent-runtime/src/context/inventory/context-limits.ts @@ -1,15 +1,13 @@ -// Context shaping limits. Keep run-level execution limits outside the context layer. +import { AGENT_RUNTIME_LIMITS } from "../../config/agent-runtime-limits.js"; -// Schema context limits -export const SCHEMA_MAX_TABLES = 20; -export const SCHEMA_MAX_COLUMNS_PER_TABLE = 50; +// Backward-compatible named exports. Definitions and environment overrides live in the central registry. +export const SCHEMA_MAX_TABLES = AGENT_RUNTIME_LIMITS.schemaMaxTables; +export const SCHEMA_MAX_COLUMNS_PER_TABLE = AGENT_RUNTIME_LIMITS.schemaMaxColumnsPerTable; -// SQL context limits -export const SQL_MAX_MODEL_ROWS = 20; -export const SQL_MAX_ACTIVITY_ROWS = 20; -export const SQL_MAX_CELL_CHARS = 500; -export const SQL_MAX_SQL_CHARS = 4000; +export const SQL_MAX_MODEL_ROWS = AGENT_RUNTIME_LIMITS.sqlMaxModelRows; +export const SQL_MAX_ACTIVITY_ROWS = AGENT_RUNTIME_LIMITS.sqlMaxActivityRows; +export const SQL_MAX_CELL_CHARS = AGENT_RUNTIME_LIMITS.sqlMaxCellChars; +export const SQL_MAX_SQL_CHARS = AGENT_RUNTIME_LIMITS.sqlMaxSqlChars; -// Per-source hard context budget. Source adapters must shape data below these limits. -export const CONTEXT_MAX_TOKENS = 32000; -export const CONTEXT_MAX_CHARS = 32000; +export const CONTEXT_MAX_TOKENS = AGENT_RUNTIME_LIMITS.contextMaxTokens; +export const CONTEXT_MAX_CHARS = AGENT_RUNTIME_LIMITS.contextMaxChars; diff --git a/packages/agent-runtime/src/context/protocol/mastra/mastra-context-budget-processor.ts b/packages/agent-runtime/src/context/protocol/mastra/mastra-context-budget-processor.ts index fb257b5..71bc13c 100644 --- a/packages/agent-runtime/src/context/protocol/mastra/mastra-context-budget-processor.ts +++ b/packages/agent-runtime/src/context/protocol/mastra/mastra-context-budget-processor.ts @@ -38,7 +38,7 @@ export type MastraContextBudgetProcessorOptions = { }; export type ContextPackageRecorder = { - record(input: { contextPackage: ContextPackage; plan: ContextPlan }): void; + record(input: { contextPackage: ContextPackage; plan?: ContextPlan }): void; }; export class MastraContextBudgetProcessor implements Processor<"context-budget"> { diff --git a/packages/agent-runtime/src/context/tool-observation/tool-observation-budget-profile.ts b/packages/agent-runtime/src/context/tool-observation/tool-observation-budget-profile.ts index 6aa8836..d5644cd 100644 --- a/packages/agent-runtime/src/context/tool-observation/tool-observation-budget-profile.ts +++ b/packages/agent-runtime/src/context/tool-observation/tool-observation-budget-profile.ts @@ -1,3 +1,4 @@ +import { AGENT_RUNTIME_LIMITS } from "../../config/agent-runtime-limits.js"; import { SCHEMA_MAX_COLUMNS_PER_TABLE, SCHEMA_MAX_TABLES, @@ -13,7 +14,7 @@ export const DEFAULT_TOOL_OBSERVATION_SOURCE_LIMIT_PROFILES: Record values.reduce((total, value) => total + value, 0); const describeOmittedTables = (omittedTables: number, names: string[]): string => { - const maxNames = 5; - const maxNameChars = 120; + const maxNames = AGENT_RUNTIME_LIMITS.toolObservationMaxNames; + const maxNameChars = AGENT_RUNTIME_LIMITS.toolObservationMaxNameChars; const preview = names.slice(0, maxNames).map((name) => name.length > maxNameChars ? `${name.slice(0, maxNameChars)}...` : name ); diff --git a/packages/agent-runtime/src/errors/tool-execution-error.ts b/packages/agent-runtime/src/errors/tool-execution-error.ts new file mode 100644 index 0000000..5f2ad23 --- /dev/null +++ b/packages/agent-runtime/src/errors/tool-execution-error.ts @@ -0,0 +1,248 @@ +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; + +export type ToolErrorCategory = + | "authorization" + | "concurrency" + | "execution" + | "protocol" + | "upstream" + | "validation"; + +export type ToolExecutionStatus = "not_started" | "failed" | "succeeded_uncommitted"; + +export type ToolRecoveryStrategy = + | "handoff" + | "refresh_and_replan" + | "refresh_and_retry" + | "request_user_input" + | "retry_after_delay" + | "retry_same_action" + | "stop" + | "use_existing_result"; + +export type ToolErrorObservation = { + ok: false; + isError: true; + error: { + code: string; + category: ToolErrorCategory; + message: string; + executionStatus: ToolExecutionStatus; + retryable: boolean; + details?: Record; + }; + recovery: { + strategy: ToolRecoveryStrategy; + instruction: string; + avoid: string[]; + }; +}; + +export type ToolExecutionErrorOptions = { + cause?: unknown; + rawResult?: unknown; +}; + +export type ToolErrorContext = { + executionStatus?: ToolExecutionStatus; + idempotency?: "required" | "supported" | "none"; + toolName: string; +}; + +/** Error carrying a stable, model-facing observation separately from internal diagnostics. */ +export class ToolExecutionError extends Error { + readonly cause: unknown; + readonly observation: ToolErrorObservation; + readonly rawResult: unknown; + + constructor(observation: ToolErrorObservation, options: ToolExecutionErrorOptions = {}) { + super(`${observation.error.code}: ${observation.error.message}`); + this.name = "ToolExecutionError"; + this.cause = options.cause; + this.observation = observation; + this.rawResult = options.rawResult; + } +} + +/** Convert any internal failure into a stable error contract that tells the model how to recover. */ +export const toToolExecutionError = ( + error: unknown, + context: ToolErrorContext, + options: Omit = {} +): ToolExecutionError => { + if (error instanceof ToolExecutionError) { + return error; + } + return new ToolExecutionError(createToolErrorObservation(error, context), { + cause: error, + ...options + }); +}; + +/** Return the public observation for an error without exposing internal stack traces or credentials. */ +export const toolErrorObservation = (error: unknown, context: ToolErrorContext): ToolErrorObservation => + error instanceof ToolExecutionError ? error.observation : createToolErrorObservation(error, context); + +const createToolErrorObservation = (error: unknown, context: ToolErrorContext): ToolErrorObservation => { + const rawMessage = errorMessage(error); + const code = errorCode(rawMessage); + const executionStatus = context.executionStatus ?? inferExecutionStatus(code); + + if (code === "ACTION_NOT_ALLOWED_IN_PHASE") { + const [, phase = "unknown", actionName = context.toolName] = rawMessage.split(":"); + return observation({ + code, + category: "protocol", + message: `Tool ${actionName} is not allowed in protocol phase ${phase}.`, + executionStatus: "not_started", + retryable: false, + strategy: "refresh_and_replan", + instruction: "Choose an action allowed in the current phase before calling this tool again.", + avoid: [`Do not repeat ${actionName} while the protocol remains in phase ${phase}.`] + }); + } + + if (code === "PROTOCOL_COMMIT_CONTENTION" || code === "PROTOCOL_REVISION_CONFLICT") { + const canRetry = context.idempotency !== "none" && executionStatus !== "not_started"; + const alreadyExecuted = executionStatus === "succeeded_uncommitted"; + return observation({ + code: "PROTOCOL_COMMIT_CONTENTION", + category: "concurrency", + message: alreadyExecuted + ? `Tool ${context.toolName} completed, but its protocol state could not be committed after concurrent updates.` + : `Tool ${context.toolName} could not update protocol state because concurrent updates did not settle.`, + executionStatus, + retryable: canRetry, + strategy: canRetry ? "refresh_and_retry" : "refresh_and_replan", + instruction: canRetry + ? "Refresh the protocol state, then retry only if the action is still needed and allowed." + : "Refresh the protocol state and replan without assuming that repeating this action is safe.", + avoid: [alreadyExecuted + ? `Do not immediately repeat ${context.toolName}; its external execution already completed.` + : `Do not retry ${context.toolName} against the stale protocol state.`] + }); + } + + if (code === "ACTION_REJECTED" || code === "PROTOCOL_GUARD_REJECTED") { + return observation({ + code, + category: code === "ACTION_REJECTED" ? "authorization" : "validation", + message: safeReason(rawMessage, `Tool ${context.toolName} was rejected before execution.`), + executionStatus: "not_started", + retryable: false, + strategy: "refresh_and_replan", + instruction: "Correct the rejected precondition or choose another allowed action.", + avoid: [`Do not repeat ${context.toolName} with unchanged input and state.`] + }); + } + + if (isSchemaValidationError(error)) { + return observation({ + code: executionStatus === "succeeded_uncommitted" ? "ACTION_RESULT_INVALID" : "ACTION_INPUT_INVALID", + category: "validation", + message: safeReason(rawMessage, `Tool ${context.toolName} returned data that did not match its contract.`), + executionStatus, + retryable: false, + strategy: "refresh_and_replan", + instruction: "Use a corrected input or an alternative tool whose contract matches the required data.", + avoid: [`Do not repeat ${context.toolName} unchanged after the same contract validation failure.`] + }); + } + + if (isUpstreamFailure(code, rawMessage)) { + return observation({ + code, + category: "upstream", + message: safeReason(rawMessage, `The upstream service used by ${context.toolName} failed.`), + executionStatus, + retryable: true, + strategy: "retry_after_delay", + instruction: "Wait briefly, then retry once; use an alternative source if the upstream failure continues.", + avoid: [`Do not loop on ${context.toolName} without delay or a retry limit.`] + }); + } + + return observation({ + code, + category: code.includes("INPUT") + || code.includes("SCHEMA") + || code.includes("NOT_FOUND") + || code.includes("UNKNOWN") + ? "validation" + : "execution", + message: safeReason(rawMessage, `Tool ${context.toolName} failed.`), + executionStatus, + retryable: false, + strategy: "refresh_and_replan", + instruction: "Use the failure reason to correct the input or choose a different action.", + avoid: [`Do not repeat ${context.toolName} unchanged after the same failure.`] + }); +}; + +const observation = (input: { + code: string; + category: ToolErrorCategory; + message: string; + executionStatus: ToolExecutionStatus; + retryable: boolean; + strategy: ToolRecoveryStrategy; + instruction: string; + avoid: string[]; +}): ToolErrorObservation => ({ + ok: false, + isError: true, + error: { + code: input.code, + category: input.category, + message: input.message, + executionStatus: input.executionStatus, + retryable: input.retryable + }, + recovery: { + strategy: input.strategy, + instruction: input.instruction, + avoid: input.avoid + } +}); + +const errorMessage = (error: unknown): string => { + if (error instanceof Error) { + return error.message; + } + if (typeof error === "string") { + return error; + } + return "ACTION_EXECUTION_FAILED"; +}; + +const errorCode = (message: string): string => { + const candidate = message.split(":", 1)[0]?.trim(); + return candidate && /^[A-Z][A-Z0-9_]+$/.test(candidate) ? candidate : "ACTION_EXECUTION_FAILED"; +}; + +const inferExecutionStatus = (code: string): ToolExecutionStatus => + code === "ACTION_NOT_ALLOWED_IN_PHASE" + || code === "ACTION_REJECTED" + || code === "PROTOCOL_GUARD_REJECTED" + || code === "CAPABILITY_ACTION_NOT_REGISTERED" + ? "not_started" + : "failed"; + +const isUpstreamFailure = (code: string, message: string): boolean => + code.includes("TIMEOUT") + || code.includes("NETWORK") + || code.startsWith("MCP_") + || /\b(fetch|socket|connection|upstream|ECONN|ETIMEDOUT)\b/i.test(message); + +const isSchemaValidationError = (error: unknown): boolean => + error instanceof Error && (error.name === "ZodError" || error.name === "ValidationError"); + +const safeReason = (message: string, fallback: string): string => { + const compact = message.replace(/\s+/g, " ").trim(); + if (!compact) { + return fallback; + } + return compact + .replace(/(api[_-]?key|authorization|token|password)\s*[=:]\s*[^\s,;]+/gi, "$1=[REDACTED]") + .slice(0, AGENT_RUNTIME_LIMITS.toolErrorMaxMessageChars); +}; diff --git a/packages/agent-runtime/src/index.ts b/packages/agent-runtime/src/index.ts index 1ce2be1..0361ecd 100644 --- a/packages/agent-runtime/src/index.ts +++ b/packages/agent-runtime/src/index.ts @@ -30,6 +30,7 @@ import { } from "@datafoundry/providers"; import { AGENT_MAX_STEPS, SQL_MAX_EXECUTION_COUNT } from "./runtime-limits.js"; +import { AGENT_RUNTIME_LIMITS } from "./config/agent-runtime-limits.js"; import { SQL_MAX_SQL_CHARS } from "./context/inventory/context-limits.js"; import { createToolObservationBoundary } from "./context/tool-observation/tool-observation-boundary.js"; import { @@ -38,6 +39,7 @@ import { import type { ContextPackageRecorder } from "./context/protocol/mastra/mastra-context-budget-processor.js"; import type { ContextPackage } from "./context/inventory/context-package.js"; import { ToolObservationDispatcher } from "./context/tool-observation/tool-observation-dispatcher.js"; +import { toolObservationModelFromPackage } from "./context/tool-observation/tool-observation-projection-items.js"; import { createAgUiContextEventSink } from "./context/protocol/ag-ui/ag-ui-context-event-sink.js"; import { NonEmptyMessageContentCompatProcessor, @@ -76,6 +78,26 @@ import type { AgentRunContext, AgentRunContextInput, AgUiEventEmitter } from "./ import { createCustomEvent } from "./events.js"; import { createTool, type ToolAction } from "@mastra/core/tools"; import { z } from "zod"; +import { + createRunProtocolBoundary, + type RunProtocolBoundary +} from "./protocol/run-protocol-boundary.js"; +import type { ProtocolClassifier, ProtocolIdentity } from "./protocol/protocol-router.js"; +import { createModelProtocolClassifier } from "./protocol/model-protocol-classifier.js"; +import { + createModelAnalysisRequirementExtractor, + type AnalysisRequirementExtractor +} from "./protocol/model-analysis-requirement-extractor.js"; +import { + createModelAnalysisContractGrounder, + type AnalysisContractGrounder +} from "./protocol/model-analysis-contract-grounder.js"; +import type { AnalysisRequirement } from "./protocol/analysis-requirements.js"; +import type { DataAnalysisState } from "./protocol/protocols/data-analysis.js"; +import { createDefaultSemanticProvider } from "./semantic/default-semantic-provider.js"; +import type { ProtocolEvent } from "./protocol/types.js"; +import type { ContextPackageRef, ProtocolStateStore } from "./protocol/types.js"; +import { toolErrorObservation as createToolErrorObservation } from "./errors/tool-execution-error.js"; export type { AgentRunContext, AgentRunContextInput, AgUiEventEmitter } from "./types.js"; export type { ContextPackage } from "./context/inventory/context-package.js"; @@ -199,6 +221,7 @@ export type CreateDataFoundryInput = { abortSignal?: AbortSignal | undefined; artifactService?: ArtifactService; contextPackageRecorder?: ContextPackageRecorder; + contextPackageExists?(reference: ContextPackageRef): boolean; dataGateway: DataGateway; emitter: AgUiEventEmitter; fileAssetService?: FileAssetService; @@ -226,6 +249,13 @@ export type CreateDataFoundryInput = { topP?: number; }; modelContextProfile?: AgentModelContextProfile; + explicitProtocol?: ProtocolIdentity; + protocolClassifier?: ProtocolClassifier; + analysisRequirementExtractor?: AnalysisRequirementExtractor; + analysisContractGrounder?: AnalysisContractGrounder; + onProtocolEvent?(event: ProtocolEvent): void; + protocolStateStore?: ProtocolStateStore; + resourceRevisions?: Record; workspaceAttachments?: WorkspaceAttachment[]; goal?: GoalRequest; /** @@ -254,6 +284,8 @@ export const createDataFoundry = async ( isolation: "bwrap" | "none" | "seatbelt"; workspaceDir: string; sessionDir: string; + protocol: RunProtocolBoundary; + flushProtocolEvents(): void; destroyWorkspace(): Promise; }> => { const toolObservationBoundary = createToolObservationBoundary({ @@ -269,6 +301,7 @@ export const createDataFoundry = async ( if (input.initialContextPackage) { contextRunState.merge(input.initialContextPackage); } + input.contextPackageRecorder?.record({ contextPackage: contextRunState.package }); const runDir = resolveWorkspaceDir({ runContext: input.runContext, @@ -330,17 +363,6 @@ export const createDataFoundry = async ( emitter: input.emitter }); }; - const governedToolFactory = new GovernedToolFactory( - dispatcher, - onGovernedResultWithSessionOutput, - registry.onGovernanceError, - { - emitter: input.emitter, - // HITL tools suspend the run and have their TOOL_CALL_RESULT emitted on interaction - // resume; the boundary must not emit a (spurious) result for them. - externallyResolvedToolNames: new Set(HITL_TOOL_NAMES) - } - ); const contextEventSink = createAgUiContextEventSink(input.emitter); const mastraContextProcessors = createMastraContextProcessorBoundary({ dispatcher, @@ -386,7 +408,7 @@ export const createDataFoundry = async ( inputSchema: z.object({ collection_id: z.string().min(1), query: z.string().min(1), - top_k: z.number().int().min(1).max(20).optional() + top_k: z.number().int().min(1).max(AGENT_RUNTIME_LIMITS.knowledgeMaxTopK).optional() }), execute: async (toolInput) => { if (!input.runContext.enabled_knowledge_ids?.includes(toolInput.collection_id)) { @@ -456,7 +478,159 @@ export const createDataFoundry = async ( ...selectedPolicyTools, ...(input.mcpTools ?? {}) }; - const tools = governedToolFactory.governTools(selectedTools); + const selectedDatasourceId = input.runContext.selected_datasource_id; + const deferredProtocolEvents: ProtocolEvent[] = []; + let protocolEventsReady = false; + const protocol = await createRunProtocolBoundary({ + runId: input.runContext.run_id, + userInput: input.runContext.user_input, + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { + packageId: contextRunState.package.packageId, + revision: contextRunState.package.revision + }, + tools: selectedTools, + ...(selectedDatasourceId + ? { + semanticProvider: createDefaultSemanticProvider({ tools: selectedTools }), + semanticRequest: { + userId: input.runContext.user_id, + workspaceId: input.runContext.workspace_id ?? "default", + datasourceId: selectedDatasourceId, + datasourceRevision: String( + input.resourceRevisions?.[`datasource:${selectedDatasourceId}`] ?? "unknown" + ) + } + } + : {}), + ...(input.explicitProtocol ? { explicitProtocol: input.explicitProtocol } : {}), + classifier: input.protocolClassifier ?? createModelProtocolClassifier(input.modelProvider), + requirementExtractor: input.analysisRequirementExtractor + ?? createModelAnalysisRequirementExtractor(input.modelProvider), + analysisContractGrounder: input.analysisContractGrounder + ?? createModelAnalysisContractGrounder(input.modelProvider), + ...(input.protocolStateStore ? { stateStore: input.protocolStateStore } : {}), + projectContext: ({ actionName, rawResult }) => { + if (isProtocolRuntimeAction(actionName)) { + const currentPackage = contextRunState.package; + return { + contextPackageRef: { + packageId: currentPackage.packageId, + revision: currentPackage.revision + }, + contextPackage: currentPackage, + observation: rawResult + }; + } + const contextPackage = dispatcher.dispatch(actionName, rawResult); + const currentPackage = contextRunState.package; + input.contextPackageRecorder?.record({ contextPackage: currentPackage }); + return { + contextPackageRef: { + packageId: currentPackage.packageId, + revision: currentPackage.revision + }, + contextPackage, + observation: toolObservationModelFromPackage(contextPackage) + }; + }, + runtimeOptions: { + ...(input.contextPackageExists ? { contextPackageExists: input.contextPackageExists } : {}), + onEvent: (event) => { + if (!protocolEventsReady) { + deferredProtocolEvents.push(event); + return false; + } + input.onProtocolEvent?.(event); + input.emitter.emit(createCustomEvent(event.type, event)); + return true; + } + } + }); + const governedToolFactory = new GovernedToolFactory( + dispatcher, + onGovernedResultWithSessionOutput, + registry.onGovernanceError, + { + actionRouter: protocol.actionRouter, + emitter: input.emitter, + externallyResolvedToolNames: new Set(HITL_TOOL_NAMES), + runId: input.runContext.run_id, + getSegmentId: () => protocol.segmentId + } + ); + const protocolState = protocol.protocolRuntime.getState(input.runContext.run_id, protocol.segmentId); + const analysisRequirements = protocolState.protocolId === "data-analysis" + ? ((protocolState.domain as DataAnalysisState).requirements ?? []).filter((requirement) => + requirement.source === "user") + : []; + const requirementsCommitTools = analysisRequirements.length > 0 + ? { + analysis_requirements_commit: createTool({ + id: "analysis_requirements_commit", + description: "Commit final claims for analysis requirements using artifact evidence from successful SQL results.", + inputSchema: z.object({ + claims: z.array(z.object({ + requirement_id: z.string().min(1), + claim: z.string().min(1), + values: z.array(z.object({ + name: z.string().min(1), + value: z.union([z.string(), z.number(), z.boolean(), z.null()]), + unit: z.string().optional() + })).max(AGENT_RUNTIME_LIMITS.requirementCommitMaxOutputFields).optional(), + evidence_refs: z.array(z.string().min(1)).optional(), + evidence_requirement_ids: z.array(z.string().min(1)).optional() + })).min(1).max(AGENT_RUNTIME_LIMITS.requirementCommitMaxClaims) + }), + execute: async (toolInput, options) => { + const toolCallId = protocolToolCallId(options); + try { + const result = await protocol.actionRouter.execute({ + runId: input.runContext.run_id, + segmentId: protocol.segmentId, + actionId: toolCallId ?? `analysis-requirements-commit:${Date.now()}`, + actionName: "analysis.requirements.commit", + input: toolInput, + idempotencyKey: toolCallId ?? JSON.stringify(toolInput) + }); + return result.observation; + } catch (error) { + return createToolErrorObservation(error, { toolName: "analysis_requirements_commit" }); + } + } + }) + } + : {}; + const tools = { + ...governedToolFactory.governTools(selectedTools), + ...requirementsCommitTools, + protocol_handoff: createTool({ + id: "protocol_handoff", + description: "Propose switching this run to another authorized protocol when the current protocol is unsuitable.", + inputSchema: z.object({ + targetProtocolId: z.string().min(1), + targetProtocolVersion: z.string().min(1), + reasonCodes: z.array(z.string().min(1)).min(1), + unresolvedGoals: z.array(z.string()) + }), + execute: async (toolInput, options) => { + const toolCallId = protocolToolCallId(options); + try { + const result = await protocol.actionRouter.execute({ + runId: input.runContext.run_id, + segmentId: protocol.segmentId, + actionId: toolCallId ?? `protocol-handoff:${Date.now()}`, + actionName: "protocol.handoff.propose", + input: toolInput, + idempotencyKey: toolCallId ?? JSON.stringify(toolInput) + }); + return result.observation; + } catch (error) { + return createToolErrorObservation(error, { toolName: "protocol_handoff" }); + } + } + }) + }; const agent = new Agent({ id: "data-foundry", name: "DataFoundry", @@ -467,8 +641,10 @@ export const createDataFoundry = async ( pythonRuntimeAvailable: Boolean(runWorkspace.pythonRuntime), selectedSkills: input.selectedSkills ?? [], taskToolsEnabled: Boolean(input.taskStateRuntime), - toolNames: Object.keys(selectedTools), + toolNames: [...Object.keys(selectedTools), ...Object.keys(requirementsCommitTools), "protocol_handoff"], mcpToolNames: input.mcpToolNames ?? [], + protocolId: protocolState.protocolId, + analysisRequirements, workspaceAttachments }), model: input.modelProvider.model as never, @@ -535,10 +711,29 @@ export const createDataFoundry = async ( return { agent: agentForAgUi, commandExecutionEnabled: runWorkspace.commandExecutionEnabled, - destroyWorkspace: () => runWorkspace.destroy(), + destroyWorkspace: async () => { + try { + await protocol.dispose(); + } finally { + await runWorkspace.destroy(); + } + }, governedMessages, ...(goalRuntime ? { goalRuntime } : {}), isolation: runWorkspace.isolation, + protocol, + flushProtocolEvents: () => { + protocolEventsReady = true; + while (deferredProtocolEvents.length > 0) { + const event = deferredProtocolEvents.shift(); + if (!event) { + continue; + } + input.onProtocolEvent?.(event); + input.emitter.emit(createCustomEvent(event.type, event)); + protocol.acknowledgeEvent(event); + } + }, workspaceDir: runWorkspace.runDir, sessionDir: runWorkspace.sessionDir }; @@ -607,6 +802,8 @@ type AgentInstructionsInput = { toolNames: string[]; /** MCP tools injected through AG-UI clientTools for this run. */ mcpToolNames: string[]; + protocolId: string; + analysisRequirements: AnalysisRequirement[]; workspaceAttachments: MaterializedWorkspaceAttachment[]; }; @@ -755,6 +952,45 @@ const buildAgentInstructions = (input: AgentInstructionsInput): string => { + "Never invent task IDs; reuse only those returned by tool results." ); } + if (input.analysisRequirements.length > 0) { + const requirementList = input.analysisRequirements.map((requirement) => { + const assertions = requirement.assertions.map((assertion) => ({ + id: assertion.id, + kind: assertion.kind, + ...(assertion.sourceTables.length > 0 ? { sourceTables: assertion.sourceTables } : {}), + ...(assertion.dimensions.length > 0 ? { dimensions: assertion.dimensions } : {}), + ...(assertion.sqlConstraints.length > 0 ? { sqlConstraints: assertion.sqlConstraints } : {}), + ...(assertion.resultChecks.length > 0 ? { resultChecks: assertion.resultChecks } : {}), + ...(assertion.claimValues.length > 0 ? { claimValues: assertion.claimValues } : {}) + })); + return `${requirement.id}: ${requirement.description}; acceptance=` + + `${requirement.acceptanceCriteria.join(" | ") || "evidenced"}; assertions=${JSON.stringify(assertions)}`; + }).join("\n"); + toolGroups.push("Analysis requirement tool: analysis_requirements_commit."); + policies.push( + "Analysis requirements are mandatory completion conditions:\n" + + requirementList + + "\nWhen using task_write, include the relevant requirement IDs in each task content. Every run_sql_readonly call " + + "must include requirement_ids for the claims it supports and expected_columns for its result contract. " + + "For every non-manual structured requirement, also include its exact assertion_ids; the runtime rejects SQL " + + "that violates declared source, aggregate, grain, filter, or time-range semantics. " + + "Include downstream validation and decision requirement IDs on the source SQL call that supplies their facts; " + + "one SQL result may support multiple requirement IDs and derived conclusions do not need duplicate queries. " + + "Preserve the user's exact metric and allocation semantics: do not substitute a requested metric with a nearby " + + "one. For counterfactual budgets, write the allocation formula before querying and validate budget conservation. " + + "A uniform percentage or rate means budget divided by the relevant cost base, then each row receives its own " + + "cost multiplied by that rate; it never means budget divided equally by row count. Use half-open timestamps to " + + "include the full requested end date. Compute threshold crossings from row-level SQL, never estimates. " + + "Do not defer every claim until the final step. As soon as a requirement has sufficient validated evidence, " + + "call analysis_requirements_commit for that requirement before starting more optional drill-downs. Commit any " + + "remaining evidenced requirements before writing final report files, and reserve the last two steps for task_check " + + "and the closing answer. Runtime resolves validated evidence already bound to that requirement, so do not guess " + + "artifact IDs. Every required claimValues entry must be copied into the claim values array with the exact " + + "verified name, numeric value, and unit; the runtime rejects unverified or mismatched values. " + + "For a derived claim, use evidence_requirement_ids to name the upstream requirement IDs that " + + "supply its facts. evidence_refs are optional hints. Missing requirements force a partial result." + ); + } if (collaborationToolsEnabled && collaborationTools.length > 0) { policies.push( "Use ask_user only when progress requires information or a decision that cannot be inferred safely. " @@ -803,6 +1039,11 @@ const buildAgentInstructions = (input: AgentInstructionsInput): string => { : ". Never use direct database clients to bypass Data Gateway.") ); } + policies.push( + `This run is governed by ${input.protocolId}@1. Use protocol_handoff only when the user's remaining goal truly ` + + "requires the other registered protocol (general-task@1 or data-analysis@1). Provide stable reasonCodes and " + + "all unresolvedGoals. Never hand off to bypass schema, SQL validation, evidence, policy, or completion gates." + ); policies.push( "Reply in the same natural language as the user's latest request. If the user mixes languages, use the dominant " + "language from the request. Keep SQL, code, table names, column names, and other technical identifiers " @@ -852,6 +1093,12 @@ const buildAgentInstructions = (input: AgentInstructionsInput): string => { + "failure plainly. " + "Do not fabricate results to mask an error." ); + policies.push( + "Recover from tool failures deliberately. Structured tool errors include error.executionStatus and a recovery " + + "object. Follow recovery.strategy and recovery.instruction, and respect every recovery.avoid item. If " + + "executionStatus is succeeded_uncommitted, do not repeat the external action unless the recovery strategy " + + "explicitly permits a retry." + ); policies.push( "Confidentiality. Never reveal credentials, datasource config, internal environment values, or workspace " + "absolute paths in your responses." @@ -1190,3 +1437,65 @@ const isWorkspaceUploadPath = (value: string): boolean => const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); + +const isProtocolRuntimeAction = (actionName: string): boolean => + actionName === "general.answer.commit" + || actionName === "protocol.handoff.propose" + || actionName.startsWith("analysis.") + || actionName.startsWith("data.query.") + || actionName === "semantic.context.resolve"; + +const protocolToolCallId = (options: unknown): string | undefined => { + if (!isRecord(options) || !isRecord(options.agent)) { + return undefined; + } + return typeof options.agent.toolCallId === "string" && options.agent.toolCallId.length > 0 + ? options.agent.toolCallId + : undefined; +}; + +export { validateProtocolDefinition } from "./protocol/definition-validator.js"; +export { ActionRouter } from "./capabilities/action-router.js"; +export { CapabilityRegistry } from "./capabilities/capability-registry.js"; +export { createToolCapabilityPlugin } from "./capabilities/tool-capability-plugin.js"; +export type * from "./capabilities/types.js"; +export { ToolExecutionError, toToolExecutionError, toolErrorObservation } from "./errors/tool-execution-error.js"; +export type * from "./errors/tool-execution-error.js"; +export { evaluateProtocolHandoff } from "./protocol/protocol-handoff.js"; +export { + createCoreAnalysisRequirements, + createUserAnalysisRequirements +} from "./protocol/analysis-requirements.js"; +export type * from "./protocol/analysis-requirements.js"; +export { + createAnalysisRequirementExtractionPrompt, + createModelAnalysisRequirementExtractor, + parseAnalysisRequirementExtractionText +} from "./protocol/model-analysis-requirement-extractor.js"; +export type { AnalysisRequirementExtractor } from "./protocol/model-analysis-requirement-extractor.js"; +export { + createAnalysisContractGroundingPrompt, + createModelAnalysisContractGrounder, + parseAnalysisContractGroundingText +} from "./protocol/model-analysis-contract-grounder.js"; +export type * from "./protocol/model-analysis-contract-grounder.js"; +export { ProtocolHandoffCoordinator } from "./protocol/protocol-handoff-coordinator.js"; +export { InMemoryProtocolStateStore } from "./protocol/in-memory-protocol-state-store.js"; +export { ProtocolRegistry } from "./protocol/protocol-registry.js"; +export { ProtocolRouter } from "./protocol/protocol-router.js"; +export { ProtocolRuntime } from "./protocol/protocol-runtime.js"; +export { createModelProtocolClassifier } from "./protocol/model-protocol-classifier.js"; +export { createRunProtocolBoundary } from "./protocol/run-protocol-boundary.js"; +export type * from "./protocol/run-protocol-boundary.js"; +export { createGeneralTaskProtocol } from "./protocol/protocols/general-task.js"; +export { createDataAnalysisProtocol } from "./protocol/protocols/data-analysis.js"; +export type * from "./protocol/protocol-handoff.js"; +export type * from "./protocol/protocol-handoff-coordinator.js"; +export type * from "./protocol/protocol-router.js"; +export type * from "./protocol/protocol-runtime.js"; +export type * from "./protocol/types.js"; +export { DataLinkSemanticProvider } from "./semantic/datalink-semantic-provider.js"; +export { LocalSemanticProvider } from "./semantic/local-semantic-provider.js"; +export { SemanticProviderChain } from "./semantic/semantic-provider-chain.js"; +export { createDefaultSemanticProvider } from "./semantic/default-semantic-provider.js"; +export type * from "./semantic/types.js"; diff --git a/packages/agent-runtime/src/protocol/analysis-contract.test.ts b/packages/agent-runtime/src/protocol/analysis-contract.test.ts new file mode 100644 index 0000000..ab81a60 --- /dev/null +++ b/packages/agent-runtime/src/protocol/analysis-contract.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; + +import { + createAnalysisAssertions, + createManualAnalysisAssertion, + resolveRequirementAssertions, + type AnalysisAssertionDraft +} from "./analysis-contract.js"; + +describe("analysis contract", () => { + it("assigns stable server-owned assertion ids and normalizes optional fields", () => { + const drafts: AnalysisAssertionDraft[] = [{ + kind: "metric", + description: "计算验证期负利润率", + sourceTables: ["orders"], + dimensions: ["region"], + sqlConstraints: [ + { kind: "aggregate", function: "count", alias: "order_count" }, + { + kind: "time_range", + column: "order_date", + start: "2023-07-01", + end: "2023-12-31", + endInclusive: true + } + ], + resultChecks: [{ kind: "non_empty", required: true }], + claimValues: [{ name: "order_count", field: "order_count", required: true, unit: "orders" }] + }]; + + expect(createAnalysisAssertions("R3", drafts)).toEqual([{ + id: "R3.A1", + requirementId: "R3", + kind: "metric", + description: "计算验证期负利润率", + required: true, + sourceTables: ["orders"], + dimensions: ["region"], + sqlConstraints: drafts[0]?.sqlConstraints, + resultChecks: drafts[0]?.resultChecks, + claimValues: drafts[0]?.claimValues + }]); + }); + + it("creates an explicit manual assertion instead of pretending an unexpressed requirement is verified", () => { + expect(createManualAnalysisAssertion("R2", "判断运营策略是否合理")).toMatchObject({ + id: "R2.A1", + requirementId: "R2", + kind: "manual", + description: "判断运营策略是否合理", + required: true, + sqlConstraints: [], + resultChecks: [], + claimValues: [] + }); + }); + + it("rejects query assertion references outside the selected requirements", () => { + const requirements = [{ + id: "R1", + assertions: createAnalysisAssertions("R1", [{ kind: "metric", description: "计算订单数" }]) + }]; + + expect(() => resolveRequirementAssertions(requirements, ["R1"], ["R2.A1"])) + .toThrow("ANALYSIS_ASSERTION_NOT_FOUND:R2.A1"); + }); + + it("requires every selected requirement to have a selected assertion", () => { + const requirements = [{ + id: "R1", + assertions: createAnalysisAssertions("R1", [{ kind: "metric", description: "计算订单数" }]) + }]; + + expect(() => resolveRequirementAssertions(requirements, ["R1"], [])) + .toThrow("ANALYSIS_ASSERTION_IDS_REQUIRED:R1"); + }); +}); diff --git a/packages/agent-runtime/src/protocol/analysis-contract.ts b/packages/agent-runtime/src/protocol/analysis-contract.ts new file mode 100644 index 0000000..f79427d --- /dev/null +++ b/packages/agent-runtime/src/protocol/analysis-contract.ts @@ -0,0 +1,266 @@ +import { z } from "zod"; + +export const ANALYSIS_ASSERTION_KINDS = [ + "metric", + "filter", + "grain", + "comparison", + "reconciliation", + "counterfactual", + "decision", + "manual" +] as const; + +export type AnalysisAssertionKind = typeof ANALYSIS_ASSERTION_KINDS[number]; +export type AnalysisScalar = string | number | boolean | null; +export type AnalysisSelector = Record>; + +export type SqlSemanticConstraint = + | { kind: "source"; table: string } + | { kind: "column"; column: string } + | { kind: "aggregate"; function: string; column?: string; alias?: string } + | { kind: "group_by"; columns: string[] } + | { kind: "filter"; column: string; operator: "eq" | "gt" | "gte" | "lt" | "lte"; value: AnalysisScalar } + | { kind: "time_range"; column: string; start: string; end: string; endInclusive: boolean }; + +export type AnalysisValueOperand = { + field?: string; + literal?: AnalysisScalar; + selector?: AnalysisSelector; +}; + +export type AnalysisResultCheck = + | { kind: "non_empty"; required: boolean } + | { kind: "row_count"; required: boolean; min?: number; max?: number } + | { kind: "not_null"; required: boolean; fields: string[] } + | { kind: "unique"; required: boolean; fields: string[] } + | { + kind: "equals" | "comparison" | "budget_conservation"; + required: boolean; + left: AnalysisValueOperand; + right: AnalysisValueOperand; + operator?: "eq" | "gt" | "gte" | "lt" | "lte"; + tolerance?: number; + } + | { + kind: "sum"; + required: boolean; + total: AnalysisValueOperand; + parts: AnalysisValueOperand[]; + tolerance?: number; + } + | { + kind: "ratio"; + required: boolean; + value: AnalysisValueOperand; + numerator: AnalysisValueOperand; + denominator: AnalysisValueOperand; + scale?: number; + tolerance?: number; + }; + +export type AnalysisClaimValueSpec = { + name: string; + field: string; + selector?: AnalysisSelector; + unit?: string; + required: boolean; + tolerance?: number; +}; + +export type AnalysisAssertionDraft = { + id?: string; + kind: AnalysisAssertionKind; + description: string; + required?: boolean; + sourceTables?: string[]; + dimensions?: string[]; + sqlConstraints?: SqlSemanticConstraint[]; + resultChecks?: AnalysisResultCheck[]; + claimValues?: AnalysisClaimValueSpec[]; +}; + +export type AnalysisAssertion = { + id: string; + requirementId: string; + kind: AnalysisAssertionKind; + description: string; + required: boolean; + sourceTables: string[]; + dimensions: string[]; + sqlConstraints: SqlSemanticConstraint[]; + resultChecks: AnalysisResultCheck[]; + claimValues: AnalysisClaimValueSpec[]; +}; + +export type AnalysisQuerySpec = { + assertionIds: string[]; +}; + +export type AnalysisValidationFinding = { + code: string; + message: string; + severity: "error" | "warning"; + assertionId?: string; +}; + +export type AnalysisVerifiedValue = { + name: string; + value: AnalysisScalar; + unit?: string; + tolerance: number; + assertionId: string; +}; + +export type AnalysisClaimValue = { + name: string; + value: AnalysisScalar; + unit?: string; +}; + +const analysisScalarSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]); +const analysisSelectorSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])); +const analysisOperandSchema = z.object({ + field: z.string().min(1).optional(), + literal: analysisScalarSchema.optional(), + selector: analysisSelectorSchema.optional() +}).refine((value) => value.field !== undefined || value.literal !== undefined, { + message: "An operand requires field or literal." +}); +const sqlConstraintSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("source"), table: z.string().min(1) }), + z.object({ + kind: z.literal("column"), + column: z.string().min(1).refine((value) => value !== "*", { + message: "Wildcard '*' is only valid as an aggregate operand." + }) + }), + z.object({ + kind: z.literal("aggregate"), + function: z.string().min(1), + column: z.string().min(1).optional(), + alias: z.string().min(1).optional() + }), + z.object({ kind: z.literal("group_by"), columns: z.array(z.string().min(1)).min(1) }), + z.object({ + kind: z.literal("filter"), + column: z.string().min(1), + operator: z.enum(["eq", "gt", "gte", "lt", "lte"]), + value: analysisScalarSchema + }), + z.object({ + kind: z.literal("time_range"), + column: z.string().min(1), + start: z.string().min(1), + end: z.string().min(1), + endInclusive: z.boolean() + }) +]); +const commonBinaryCheckFields = { + required: z.boolean(), + left: analysisOperandSchema, + right: analysisOperandSchema, + tolerance: z.number().nonnegative().optional() +}; +const resultCheckSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("non_empty"), required: z.boolean() }), + z.object({ + kind: z.literal("row_count"), + required: z.boolean(), + min: z.number().int().nonnegative().optional(), + max: z.number().int().nonnegative().optional() + }), + z.object({ kind: z.literal("not_null"), required: z.boolean(), fields: z.array(z.string().min(1)).min(1) }), + z.object({ kind: z.literal("unique"), required: z.boolean(), fields: z.array(z.string().min(1)).min(1) }), + z.object({ kind: z.literal("equals"), ...commonBinaryCheckFields }), + z.object({ + kind: z.literal("sum"), + required: z.boolean(), + total: analysisOperandSchema, + parts: z.array(analysisOperandSchema).min(1) + }), + z.object({ + kind: z.literal("comparison"), + ...commonBinaryCheckFields, + operator: z.enum(["eq", "gt", "gte", "lt", "lte"]) + }), + z.object({ kind: z.literal("budget_conservation"), ...commonBinaryCheckFields }), + z.object({ + kind: z.literal("ratio"), + required: z.boolean(), + value: analysisOperandSchema, + numerator: analysisOperandSchema, + denominator: analysisOperandSchema, + scale: z.number().optional(), + tolerance: z.number().nonnegative().optional() + }) +]); +const claimValueSpecSchema = z.object({ + name: z.string().min(1), + field: z.string().min(1), + selector: analysisSelectorSchema.optional(), + unit: z.string().min(1).optional(), + required: z.boolean(), + tolerance: z.number().nonnegative().optional() +}); + +export const analysisAssertionDraftSchema = z.object({ + id: z.string().optional(), + kind: z.enum(ANALYSIS_ASSERTION_KINDS), + description: z.string().min(1).max(500), + required: z.boolean().optional(), + sourceTables: z.array(z.string().min(1)).optional(), + dimensions: z.array(z.string().min(1)).optional(), + sqlConstraints: z.array(sqlConstraintSchema).optional(), + resultChecks: z.array(resultCheckSchema).optional(), + claimValues: z.array(claimValueSpecSchema).optional() +}).strict(); + +/** Assign stable server-owned IDs and normalize model-proposed analysis assertions. */ +export const createAnalysisAssertions = ( + requirementId: string, + drafts: AnalysisAssertionDraft[] +): AnalysisAssertion[] => drafts.map((draft, index) => ({ + id: `${requirementId}.A${index + 1}`, + requirementId, + kind: draft.kind, + description: draft.description.trim(), + required: draft.required !== false, + sourceTables: uniqueStrings(draft.sourceTables ?? []), + dimensions: uniqueStrings(draft.dimensions ?? []), + sqlConstraints: structuredClone(draft.sqlConstraints ?? []), + resultChecks: structuredClone(draft.resultChecks ?? []), + claimValues: structuredClone(draft.claimValues ?? []) +})); + +/** Represent an unexpressed requirement explicitly instead of claiming deterministic verification. */ +export const createManualAnalysisAssertion = ( + requirementId: string, + description: string +): AnalysisAssertion => createAnalysisAssertions(requirementId, [{ + kind: "manual", + description +}])[0] as AnalysisAssertion; + +/** Resolve selected assertion IDs and reject cross-requirement or incomplete references. */ +export const resolveRequirementAssertions = ( + requirements: Array<{ id: string; assertions: AnalysisAssertion[] }>, + requirementIds: string[], + assertionIds: string[] +): AnalysisAssertion[] => { + const selectedRequirements = requirements.filter((requirement) => requirementIds.includes(requirement.id)); + const allowedAssertions = selectedRequirements.flatMap((requirement) => requirement.assertions); + for (const assertionId of assertionIds) { + if (!allowedAssertions.some((assertion) => assertion.id === assertionId)) { + throw new Error(`ANALYSIS_ASSERTION_NOT_FOUND:${assertionId}`); + } + } + for (const requirement of selectedRequirements) { + if (!assertionIds.some((assertionId) => requirement.assertions.some((assertion) => assertion.id === assertionId))) { + throw new Error(`ANALYSIS_ASSERTION_IDS_REQUIRED:${requirement.id}`); + } + } + return allowedAssertions.filter((assertion) => assertionIds.includes(assertion.id)); +}; + +const uniqueStrings = (values: string[]): string[] => [...new Set(values.map((value) => value.trim()).filter(Boolean))]; diff --git a/packages/agent-runtime/src/protocol/analysis-requirements.ts b/packages/agent-runtime/src/protocol/analysis-requirements.ts new file mode 100644 index 0000000..e4f2864 --- /dev/null +++ b/packages/agent-runtime/src/protocol/analysis-requirements.ts @@ -0,0 +1,139 @@ +import { + createAnalysisAssertions, + createManualAnalysisAssertion, + type AnalysisAssertion, + type AnalysisAssertionDraft, + type AnalysisClaimValue, + type AnalysisValidationFinding, + type AnalysisVerifiedValue +} from "./analysis-contract.js"; + +export const ANALYSIS_REQUIREMENT_KINDS = [ + "data_quality", + "metric", + "segmentation", + "comparison", + "validation", + "counterfactual", + "decision", + "deliverable" +] as const; + +export type AnalysisRequirementKind = typeof ANALYSIS_REQUIREMENT_KINDS[number]; +export type AnalysisRequirementStatus = "pending" | "queried" | "validated" | "evidenced" | "reported"; + +export type AnalysisRequirement = { + id: string; + kind: AnalysisRequirementKind; + description: string; + acceptanceCriteria: string[]; + assertions: AnalysisAssertion[]; + required: boolean; + source: "protocol" | "user"; + status: AnalysisRequirementStatus; + taskIds: string[]; + queryAttemptIds: string[]; + evidenceBindingIds: string[]; + reportedClaimIds: string[]; +}; + +export type AnalysisQueryAttempt = { + id: string; + requirementIds: string[]; + assertionIds: string[]; + assertions: AnalysisAssertion[]; + sql?: string; + expectedColumns: string[]; + status: "planned" | "validated" | "executed" | "evidenced"; + valid: boolean; + artifactId?: string; + auditLogId?: string; + resultFields: string[]; + validationFindings: AnalysisValidationFinding[]; + resultValidationFindings: AnalysisValidationFinding[]; + verifiedValues: AnalysisVerifiedValue[]; +}; + +export type AnalysisEvidenceBinding = { + id: string; + requirementId: string; + queryAttemptId: string; + artifactId: string; + auditLogId: string; + resultFields: string[]; + validationStatus: "passed"; +}; + +export type AnalysisReportedClaim = { + id: string; + requirementId: string; + claim: string; + evidenceBindingIds: string[]; + values: AnalysisClaimValue[]; +}; + +export type TaskRequirementLink = { + taskId: string; + requirementIds: string[]; +}; + +export type AnalysisRequirementDraft = { + kind: AnalysisRequirementKind; + description: string; + acceptanceCriteria: string[]; + assertions?: AnalysisAssertionDraft[]; +}; + +/** Create deterministic protocol-owned requirements that models cannot remove. */ +export const createCoreAnalysisRequirements = (): AnalysisRequirement[] => [ + createRequirement("CORE_SCHEMA", "validation", "Inspect and ground the physical schema", "protocol"), + createRequirement("CORE_SEMANTIC", "validation", "Resolve semantic context or disclose fallback", "protocol"), + createRequirement("CORE_QUERY", "validation", "Validate the current read-only query", "protocol"), + createRequirement("CORE_RESULT", "validation", "Validate the current query result", "protocol"), + createRequirement("CORE_EVIDENCE", "validation", "Bind audited evidence for the current result", "protocol") +]; + +/** Normalize model drafts into stable server-owned user requirement records. */ +export const createUserAnalysisRequirements = (drafts: AnalysisRequirementDraft[]): AnalysisRequirement[] => { + const seen = new Set(); + const uniqueDrafts: AnalysisRequirementDraft[] = []; + for (const draft of drafts) { + const description = draft.description.trim(); + const key = description.toLocaleLowerCase(); + if (!description || seen.has(key)) { + continue; + } + seen.add(key); + uniqueDrafts.push({ ...draft, description }); + } + return uniqueDrafts.map((draft, index) => { + const id = `R${index + 1}`; + return { + ...createRequirement(id, draft.kind, draft.description, "user"), + acceptanceCriteria: [...draft.acceptanceCriteria], + assertions: draft.assertions && draft.assertions.length > 0 + ? createAnalysisAssertions(id, draft.assertions) + : [createManualAnalysisAssertion(id, draft.description)] + }; + }); +}; + +const createRequirement = ( + id: string, + kind: AnalysisRequirementKind, + description: string, + source: AnalysisRequirement["source"] +): AnalysisRequirement => ({ + id, + kind, + description, + acceptanceCriteria: [], + assertions: [], + required: true, + source, + status: "pending", + taskIds: [], + queryAttemptIds: [], + evidenceBindingIds: [], + reportedClaimIds: [] +}); diff --git a/packages/agent-runtime/src/protocol/definition-validator.test.ts b/packages/agent-runtime/src/protocol/definition-validator.test.ts new file mode 100644 index 0000000..2bdbb27 --- /dev/null +++ b/packages/agent-runtime/src/protocol/definition-validator.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; + +import { validateProtocolDefinition } from "./definition-validator.js"; + +describe("validateProtocolDefinition", () => { + it("rejects a missing initial phase", () => { + expect(() => validateProtocolDefinition({ + id: "test/protocol", + version: "1", + initialPhase: "missing", + phases: { + active: { + allowedActions: [], + transitions: [] + } + }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) + })).toThrow("PROTOCOL_INITIAL_PHASE_NOT_FOUND:test/protocol@1:missing"); + }); + + it("rejects a transition to an unknown phase", () => { + expect(() => validateProtocolDefinition({ + id: "test/protocol", + version: "1", + initialPhase: "active", + phases: { + active: { + allowedActions: ["test.read"], + transitions: [{ targetPhase: "missing", when: () => true }] + } + }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) + })).toThrow("PROTOCOL_TRANSITION_TARGET_NOT_FOUND:test/protocol@1:active:missing"); + }); + + it("rejects a duplicate action within one phase", () => { + expect(() => validateProtocolDefinition({ + id: "test/protocol", + version: "1", + initialPhase: "active", + phases: { + active: { + allowedActions: ["test.read", "test.read"], + transitions: [] + } + }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) + })).toThrow("PROTOCOL_DUPLICATE_PHASE_ACTION:test/protocol@1:active:test.read"); + }); + + it("accepts an explicitly cyclic protocol", () => { + expect(() => validateProtocolDefinition({ + id: "test/cyclic", + version: "1", + initialPhase: "inspect", + phases: { + inspect: { + allowedActions: ["data.inspect"], + transitions: [{ targetPhase: "query", when: () => true }] + }, + query: { + allowedActions: ["data.query"], + transitions: [{ targetPhase: "inspect", when: () => true }] + } + }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) + })).not.toThrow(); + }); +}); diff --git a/packages/agent-runtime/src/protocol/definition-validator.ts b/packages/agent-runtime/src/protocol/definition-validator.ts new file mode 100644 index 0000000..3d59cc0 --- /dev/null +++ b/packages/agent-runtime/src/protocol/definition-validator.ts @@ -0,0 +1,29 @@ +import type { AgentProtocolDefinition } from "./types.js"; + +/** Validate the structural invariants required to register a protocol definition. */ +export const validateProtocolDefinition = (definition: AgentProtocolDefinition): void => { + if (!Object.hasOwn(definition.phases, definition.initialPhase)) { + throw new Error( + `PROTOCOL_INITIAL_PHASE_NOT_FOUND:${definition.id}@${definition.version}:${definition.initialPhase}` + ); + } + for (const [phaseName, phase] of Object.entries(definition.phases)) { + const allowedActions = new Set(); + for (const actionName of phase.allowedActions) { + if (allowedActions.has(actionName)) { + throw new Error( + `PROTOCOL_DUPLICATE_PHASE_ACTION:${definition.id}@${definition.version}:${phaseName}:${actionName}` + ); + } + allowedActions.add(actionName); + } + for (const transition of phase.transitions) { + if (!Object.hasOwn(definition.phases, transition.targetPhase)) { + throw new Error( + `PROTOCOL_TRANSITION_TARGET_NOT_FOUND:${definition.id}@${definition.version}:${phaseName}:` + + transition.targetPhase + ); + } + } + } +}; diff --git a/packages/agent-runtime/src/protocol/in-memory-protocol-state-store.ts b/packages/agent-runtime/src/protocol/in-memory-protocol-state-store.ts new file mode 100644 index 0000000..069b727 --- /dev/null +++ b/packages/agent-runtime/src/protocol/in-memory-protocol-state-store.ts @@ -0,0 +1,94 @@ +import type { ProtocolEvent, ProtocolRunState, ProtocolStateStore } from "./types.js"; + +export class InMemoryProtocolStateStore implements ProtocolStateStore { + private readonly states = new Map(); + private readonly currentSegments = new Map(); + private readonly events = new Map(); + + create( + state: ProtocolRunState, + events: ProtocolEvent[] = [] + ): ProtocolRunState { + const key = protocolStateKey(state.runId, state.segmentId); + if (this.states.has(key)) { + throw new Error(`PROTOCOL_SEGMENT_ALREADY_STARTED:${state.runId}:${state.segmentId}`); + } + this.states.set(key, state as ProtocolRunState); + this.currentSegments.set(state.runId, state.segmentId); + this.recordEvents(events); + return state; + } + + get(runId: string, segmentId?: string): ProtocolRunState { + const state = this.find(runId, segmentId); + if (!state) { + throw new Error(`PROTOCOL_RUN_NOT_FOUND:${runId}`); + } + return state; + } + + find(runId: string, segmentId?: string): ProtocolRunState | undefined { + const resolvedSegmentId = segmentId ?? this.currentSegments.get(runId); + const state = resolvedSegmentId ? this.states.get(protocolStateKey(runId, resolvedSegmentId)) : undefined; + return state as ProtocolRunState | undefined; + } + + compareAndSet( + state: ProtocolRunState, + expectedRevision: number, + events: ProtocolEvent[] = [] + ): ProtocolRunState { + const current = this.get(state.runId, state.segmentId); + if (current.revision !== expectedRevision) { + throw new Error( + `PROTOCOL_REVISION_CONFLICT:${state.runId}:${state.segmentId}:${expectedRevision}:${current.revision}` + ); + } + this.states.set(protocolStateKey(state.runId, state.segmentId), state as ProtocolRunState); + this.recordEvents(events); + return state; + } + + transitionSegment(input: { + current: ProtocolRunState; + expectedRevision: number; + next: ProtocolRunState; + events?: ProtocolEvent[]; + }): { + current: ProtocolRunState; + next: ProtocolRunState; + } { + const current = this.get(input.current.runId, input.current.segmentId); + if (current.revision !== input.expectedRevision) { + throw new Error( + `PROTOCOL_REVISION_CONFLICT:${current.runId}:${current.segmentId}:` + + `${input.expectedRevision}:${current.revision}` + ); + } + const nextKey = protocolStateKey(input.next.runId, input.next.segmentId); + if (this.states.has(nextKey)) { + throw new Error(`PROTOCOL_SEGMENT_ALREADY_STARTED:${input.next.runId}:${input.next.segmentId}`); + } + this.states.set(protocolStateKey(input.current.runId, input.current.segmentId), input.current as ProtocolRunState); + this.states.set(nextKey, input.next as ProtocolRunState); + this.currentSegments.set(input.next.runId, input.next.segmentId); + this.recordEvents(input.events ?? []); + return input; + } + + acknowledgeEvent(event: ProtocolEvent): void { + this.events.delete(event.eventId); + } + + pendingEvents(runId: string): ProtocolEvent[] { + return [...this.events.values()].filter((event) => event.runId === runId); + } + + private recordEvents(events: ProtocolEvent[]): void { + for (const event of events) { + this.events.set(event.eventId, event); + } + } +} + +const protocolStateKey = (runId: string, segmentId: string): string => `${runId}:${segmentId}`; diff --git a/packages/agent-runtime/src/protocol/model-analysis-contract-grounder.test.ts b/packages/agent-runtime/src/protocol/model-analysis-contract-grounder.test.ts new file mode 100644 index 0000000..d043753 --- /dev/null +++ b/packages/agent-runtime/src/protocol/model-analysis-contract-grounder.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it } from "vitest"; + +import { createUserAnalysisRequirements } from "./analysis-requirements.js"; +import { + createFallbackAnalysisContractGrounding, + createAnalysisContractGroundingPrompt, + createAnalysisContractGroundingRetryInstruction, + parseAnalysisContractGroundingText +} from "./model-analysis-contract-grounder.js"; + +const requirements = createUserAnalysisRequirements([{ + kind: "metric", + description: "统计物流单量和日期范围", + acceptanceCriteria: ["报告总单量、最早日期和最晚日期"] +}]); + +const physicalSchema = { + schema_id: "schema-1", + tables: [{ + name: "dacomp-zh-006", + columns: [ + { name: "日期", type: "DATE" }, + { name: "物流单号", type: "VARCHAR" } + ] + }] +}; + +describe("analysis contract grounding", () => { + it("grounds assertions against the inspected physical schema with server-owned ids", () => { + const result = parseAnalysisContractGroundingText(JSON.stringify({ + contracts: [{ + requirementId: "R1", + assertions: [{ + kind: "metric", + description: "物流单量与日期边界", + sourceTables: ["dacomp-zh-006"], + sqlConstraints: [ + { kind: "source", table: "dacomp-zh-006" }, + { kind: "column", column: "日期" }, + { kind: "aggregate", function: "COUNT", column: "物流单号", alias: "shipment_count" }, + { kind: "aggregate", function: "MIN", column: "日期", alias: "min_date" }, + { kind: "aggregate", function: "MAX", column: "日期", alias: "max_date" } + ], + resultChecks: [{ + kind: "not_null", + required: true, + fields: ["shipment_count", "min_date", "max_date"] + }], + claimValues: [ + { name: "shipment_count", field: "shipment_count", required: true }, + { name: "min_date", field: "min_date", required: true }, + { name: "max_date", field: "max_date", required: true } + ] + }] + }] + }), requirements, physicalSchema); + + expect(result.findings).toEqual([]); + expect(result.requirements[0]).toMatchObject({ + id: "R1", + description: "统计物流单量和日期范围", + assertions: [{ + id: "R1.A1", + requirementId: "R1", + kind: "metric", + sourceTables: ["dacomp-zh-006"] + }] + }); + }); + + it("downgrades invented physical identifiers instead of making them authoritative", () => { + const result = parseAnalysisContractGroundingText(JSON.stringify({ + contracts: [{ + requirementId: "R1", + assertions: [{ + kind: "metric", + description: "订单数", + sourceTables: ["orders"], + sqlConstraints: [{ kind: "column", column: "order_date" }], + claimValues: [{ name: "order_count", field: "order_count", required: true }] + }] + }] + }), requirements, physicalSchema); + + expect(result.requirements[0]?.assertions).toEqual([ + expect.objectContaining({ id: "R1.A1", kind: "manual", sourceTables: [], sqlConstraints: [] }) + ]); + expect(result.findings).toEqual(expect.arrayContaining([ + expect.objectContaining({ requirementId: "R1", code: "CONTRACT_UNKNOWN_TABLE" }), + expect.objectContaining({ requirementId: "R1", code: "CONTRACT_UNKNOWN_COLUMN" }) + ])); + }); + + it("rejects structured assertions that cannot produce runtime-verified claim values", () => { + const output = JSON.stringify({ + contracts: [{ + requirementId: "R1", + assertions: [{ + kind: "metric", + description: "物流单量", + sourceTables: ["dacomp-zh-006"], + sqlConstraints: [{ + kind: "aggregate", + function: "COUNT", + column: "物流单号", + alias: "shipment_count" + }], + resultChecks: [{ kind: "not_null", required: true, fields: ["shipment_count"] }] + }] + }] + }); + + expect(() => parseAnalysisContractGroundingText(output, requirements, physicalSchema)) + .toThrow("Structured assertions require at least one claimValues entry"); + }); + + it("rejects a wildcard used as a standalone physical column constraint", () => { + const output = JSON.stringify({ + contracts: [{ + requirementId: "R1", + assertions: [{ + kind: "metric", + description: "物流单量", + sourceTables: ["dacomp-zh-006"], + sqlConstraints: [ + { kind: "column", column: "*" }, + { kind: "aggregate", function: "COUNT", column: "*", alias: "shipment_count" } + ], + claimValues: [{ name: "shipment_count", field: "shipment_count", required: true }] + }] + }] + }); + + expect(() => parseAnalysisContractGroundingText(output, requirements, physicalSchema)) + .toThrow("Wildcard '*' is only valid as an aggregate operand"); + }); + + it("makes schema and semantic grounding constraints explicit in the model prompt", () => { + const prompt = createAnalysisContractGroundingPrompt({ + requirements, + physicalSchema, + semanticResolution: { + provider: "datalink", + mode: "live", + datasourceRevision: "revision-7", + value: { definitions: ["物流单号唯一标识一票物流"] }, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [] + }, + datasourceRevision: "revision-7" + }); + + expect(prompt).toContain("dacomp-zh-006"); + expect(prompt).toContain("物流单号"); + expect(prompt).toContain("revision-7"); + expect(prompt).toContain("不得发明表名或字段名"); + }); + + it("defines the exact nested JSON shapes required from the model", () => { + const prompt = createAnalysisContractGroundingPrompt({ + requirements, + physicalSchema, + semanticResolution: { + provider: "local", + mode: "fallback", + datasourceRevision: "revision-8", + trust: "verified", + value: { tables: ["dacomp-zh-006"] }, + capabilities: ["physical-schema"], + warnings: [] + }, + datasourceRevision: "revision-8" + }); + + expect(prompt).toContain('{"kind":"source","table":""}'); + expect(prompt).toContain('{"kind":"aggregate","function":"COUNT","column":"*","alias":"row_count"}'); + expect(prompt).toContain('{"kind":"not_null","required":true,"fields":["row_count"]}'); + expect(prompt).toContain('{"name":"row_count","field":"row_count","required":true}'); + expect(prompt).toContain("每个 sqlConstraints 元素只能表达一种 kind"); + expect(prompt).toContain('"*" 只能作为 aggregate 的 column'); + }); + + it("turns validation failures into precise retry instructions", () => { + let parseError: unknown; + try { + parseAnalysisContractGroundingText(JSON.stringify({ + contracts: [{ + requirementId: "R1", + assertions: [{ + kind: "metric", + description: "物流单量", + sourceTables: ["dacomp-zh-006"], + sqlConstraints: [{ source: "dacomp-zh-006", aggregate: "COUNT(*)" }], + resultChecks: ["结果必须为非负整数"] + }] + }] + }), requirements, physicalSchema); + } catch (error) { + parseError = error; + } + + const instruction = createAnalysisContractGroundingRetryInstruction(parseError); + + expect(instruction).toContain("contracts.0.assertions.0.sqlConstraints.0.kind"); + expect(instruction).toContain("Invalid discriminator value"); + expect(instruction).toContain("不要重复原来的无效结构"); + }); + + it("preserves the final validation reason when grounding falls back to manual", () => { + const parseError = new Error("contracts.0.assertions.0.claimValues.0.required is missing"); + + const result = createFallbackAnalysisContractGrounding(requirements, parseError); + + expect(result.requirements[0]?.assertions).toEqual([ + expect.objectContaining({ kind: "manual" }) + ]); + expect(result.findings).toEqual([ + expect.objectContaining({ + requirementId: "R1", + code: "CONTRACT_INVALID_OUTPUT", + message: expect.stringContaining("claimValues.0.required is missing") + }) + ]); + }); +}); diff --git a/packages/agent-runtime/src/protocol/model-analysis-contract-grounder.ts b/packages/agent-runtime/src/protocol/model-analysis-contract-grounder.ts new file mode 100644 index 0000000..05bafbd --- /dev/null +++ b/packages/agent-runtime/src/protocol/model-analysis-contract-grounder.ts @@ -0,0 +1,309 @@ +import { Agent } from "@mastra/core/agent"; +import type { ModelProvider } from "@datafoundry/providers"; +import { z } from "zod"; + +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; +import { + analysisAssertionDraftSchema, + createAnalysisAssertions, + createManualAnalysisAssertion, + type AnalysisAssertionDraft, + type SqlSemanticConstraint +} from "./analysis-contract.js"; +import type { AnalysisRequirement } from "./analysis-requirements.js"; +import type { SemanticResolution } from "../semantic/types.js"; + +const groundedContractSchema = z.object({ + requirementId: z.string().min(1), + assertions: z.array(analysisAssertionDraftSchema).min(1).max(12) +}).strict().superRefine((contract, context) => { + contract.assertions.forEach((assertion, index) => { + if (assertion.kind !== "manual" && (assertion.claimValues?.length ?? 0) === 0) { + context.addIssue({ + code: "custom", + message: "Structured assertions require at least one claimValues entry.", + path: ["assertions", index, "claimValues"] + }); + } + }); +}); + +const groundingSchema = z.object({ + contracts: z.array(groundedContractSchema).max(16) +}).strict(); + +export type AnalysisContractGroundingInput = { + requirements: AnalysisRequirement[]; + physicalSchema: unknown; + semanticResolution: SemanticResolution; + datasourceRevision: string; +}; + +export type AnalysisContractGroundingFinding = { + requirementId: string; + code: "CONTRACT_MISSING" | "CONTRACT_INVALID_OUTPUT" | "CONTRACT_UNKNOWN_TABLE" | "CONTRACT_UNKNOWN_COLUMN"; + message: string; +}; + +export type AnalysisContractGroundingResult = { + requirements: AnalysisRequirement[]; + findings: AnalysisContractGroundingFinding[]; +}; + +export type AnalysisContractGrounder = ( + input: AnalysisContractGroundingInput +) => Promise; + +/** Build a schema-bound prompt for converting logical requirements into physical assertions. */ +export const createAnalysisContractGroundingPrompt = (input: AnalysisContractGroundingInput): string => [ + "你是数据分析 Contract grounding 器，不执行 SQL，也不回答用户问题。", + "根据已检查的物理 schema 和语义解析结果，把每个用户 requirement 转为可验证的结构化 assertions。", + "不得发明表名或字段名；sourceTables、dimensions 和 sqlConstraints 中的物理标识必须逐字存在于 schema。", + "语义信息只用于选择正确字段和解释口径，不能覆盖物理 schema。没有足够依据时使用 kind=manual。", + `datasourceRevision: ${input.datasourceRevision}`, + `requirements: ${JSON.stringify(logicalRequirements(input.requirements))}`, + `physicalSchema: ${JSON.stringify(input.physicalSchema)}`, + `semanticResolution: ${JSON.stringify(input.semanticResolution)}`, + "只返回合法 JSON 对象，不要 Markdown。顶层字段为 contracts。", + "contracts 每项包含 requirementId 和 assertions；requirementId 必须来自输入，不得生成新 ID。", + "每个 assertion 必须包含 kind 和 description；可选字段为 sourceTables、dimensions、sqlConstraints、", + "resultChecks、claimValues。所有复数字段都必须是数组。", + "sqlConstraints 仅可使用 source、column、aggregate、group_by、filter、time_range 结构。", + "每个 sqlConstraints 元素只能表达一种 kind，不得把 source、column、aggregate 合并到同一对象。", + '合法 sqlConstraints 元素示例: {"kind":"source","table":""}, ' + + '{"kind":"column","column":""}, ' + + '{"kind":"aggregate","function":"COUNT","column":"*","alias":"row_count"}。', + '"*" 只能作为 aggregate 的 column，绝不能用于 kind=column、dimensions、filter 或 time_range。', + '合法 resultChecks 元素示例: {"kind":"non_empty","required":true}, ' + + '{"kind":"not_null","required":true,"fields":["row_count"]}。不得使用字符串或 type 字段。', + '合法 claimValues 元素示例: {"name":"row_count","field":"row_count","required":true}。' + + "name、field、required 缺一不可。", + "每个非 manual assertion 至少包含一个 claimValues 元素，否则无法把 SQL 结果绑定为运行时验证值。", + "尖括号中的示例物理标识必须替换为 physicalSchema 中逐字存在的真实标识，不得原样输出。", + "不要为 assertion 生成 id，服务端会分配稳定 ID。" +].join("\n"); + +/** Parse model grounding output, reject invented identifiers, and preserve requirement state. */ +export const parseAnalysisContractGroundingText = ( + text: string, + requirements: AnalysisRequirement[], + physicalSchema: unknown +): AnalysisContractGroundingResult => { + const trimmed = text.trim(); + const unfenced = trimmed.startsWith("```") + ? trimmed.replace(/^```(?:json)?\s*/iu, "").replace(/\s*```$/u, "") + : trimmed; + const parsed = groundingSchema.parse(JSON.parse(unfenced) as unknown); + const contracts = new Map(parsed.contracts.map((contract) => [ + contract.requirementId, + contract.assertions.map(normalizeAssertionDraft) + ])); + const schema = inspectPhysicalSchema(physicalSchema); + const findings: AnalysisContractGroundingFinding[] = []; + const grounded = requirements.map((requirement) => { + if (requirement.source !== "user") { + return cloneRequirement(requirement); + } + const drafts = contracts.get(requirement.id); + if (!drafts) { + findings.push({ + requirementId: requirement.id, + code: "CONTRACT_MISSING", + message: `No grounded contract was returned for ${requirement.id}.` + }); + return withManualAssertion(requirement); + } + const draftFindings = validateAssertionDrafts(requirement.id, drafts, schema); + findings.push(...draftFindings); + return draftFindings.length > 0 + ? withManualAssertion(requirement) + : { ...cloneRequirement(requirement), assertions: createAnalysisAssertions(requirement.id, drafts) }; + }); + return { requirements: grounded, findings }; +}; + +/** Convert a parser failure into concrete model-facing repair instructions for the next grounding attempt. */ +export const createAnalysisContractGroundingRetryInstruction = (error: unknown): string => { + return [ + "", + "上次输出未通过服务端 schema 校验。不要重复原来的无效结构，只修正以下问题：", + describeAnalysisContractGroundingError(error), + "重新检查每个嵌套对象的 kind 和必填字段，并严格返回紧凑 JSON。" + ].join("\n"); +}; + +/** Preserve an exhausted grounding failure as an explicit manual contract with actionable findings. */ +export const createFallbackAnalysisContractGrounding = ( + requirements: AnalysisRequirement[], + error?: unknown +): AnalysisContractGroundingResult => ({ + requirements: requirements.map((requirement) => requirement.source === "user" + ? withManualAssertion(requirement) + : cloneRequirement(requirement)), + findings: requirements + .filter((requirement) => requirement.source === "user") + .map((requirement) => ({ + requirementId: requirement.id, + code: error === undefined ? "CONTRACT_MISSING" as const : "CONTRACT_INVALID_OUTPUT" as const, + message: error === undefined + ? `No valid grounded contract was produced for ${requirement.id}.` + : `Grounding output was invalid for ${requirement.id}: ${describeAnalysisContractGroundingError(error)}` + })) +}); + +/** Create a tool-free model grounder backed by the configured run model. */ +export const createModelAnalysisContractGrounder = ( + provider: Exclude +): AnalysisContractGrounder => { + const agent = new Agent({ + id: "analysis-contract-grounder", + name: "Analysis Contract Grounder", + instructions: "Ground analysis requirements against supplied schema only. Never execute or answer the task.", + model: provider.model as never + }); + return async (input) => { + const prompt = createAnalysisContractGroundingPrompt(input); + let retryInstruction = ""; + let lastError: unknown; + for (let attempt = 0; attempt < AGENT_RUNTIME_LIMITS.contractGrounderMaxAttempts; attempt += 1) { + const output = await agent.generate(`${prompt}${retryInstruction}`, { + maxSteps: AGENT_RUNTIME_LIMITS.modelHelperMaxSteps, + modelSettings: { + maxOutputTokens: AGENT_RUNTIME_LIMITS.contractGrounderMaxOutputTokens, + temperature: 0 + } + }); + try { + return parseAnalysisContractGroundingText(output.text, input.requirements, input.physicalSchema); + } catch (error) { + lastError = error; + retryInstruction = createAnalysisContractGroundingRetryInstruction(error); + } + } + return createFallbackAnalysisContractGrounding(input.requirements, lastError); + }; +}; + +const logicalRequirements = (requirements: AnalysisRequirement[]): unknown[] => requirements + .filter((requirement) => requirement.source === "user") + .map((requirement) => ({ + id: requirement.id, + kind: requirement.kind, + description: requirement.description, + acceptanceCriteria: requirement.acceptanceCriteria + })); + +const normalizeAssertionDraft = ( + draft: z.infer +): AnalysisAssertionDraft => structuredClone(draft) as AnalysisAssertionDraft; + +const describeAnalysisContractGroundingError = (error: unknown): string => error instanceof z.ZodError + ? error.issues.map((issue) => `${issue.path.join(".") || "root"}: ${issue.message}`).join("; ") + : error instanceof Error + ? error.message + : String(error ?? "unknown validation error"); + +const withManualAssertion = (requirement: AnalysisRequirement): AnalysisRequirement => ({ + ...cloneRequirement(requirement), + assertions: [createManualAnalysisAssertion(requirement.id, requirement.description)] +}); + +const cloneRequirement = (requirement: AnalysisRequirement): AnalysisRequirement => ({ + ...requirement, + acceptanceCriteria: [...requirement.acceptanceCriteria], + assertions: structuredClone(requirement.assertions), + taskIds: [...requirement.taskIds], + queryAttemptIds: [...requirement.queryAttemptIds], + evidenceBindingIds: [...requirement.evidenceBindingIds], + reportedClaimIds: [...requirement.reportedClaimIds] +}); + +type PhysicalSchemaIndex = { tables: Set; columns: Set }; + +const inspectPhysicalSchema = (physicalSchema: unknown): PhysicalSchemaIndex => { + const tables = new Set(); + const columns = new Set(); + const rawTables = recordValue(physicalSchema, "tables"); + if (!Array.isArray(rawTables)) { + return { tables, columns }; + } + for (const rawTable of rawTables) { + const tableName = typeof rawTable === "string" ? rawTable : recordString(rawTable, "name", "table_name"); + if (tableName) { + tables.add(normalizeIdentifier(tableName)); + } + const rawColumns = recordValue(rawTable, "columns"); + if (!Array.isArray(rawColumns)) { + continue; + } + for (const rawColumn of rawColumns) { + const columnName = typeof rawColumn === "string" ? rawColumn : recordString(rawColumn, "name", "column_name"); + if (columnName) { + columns.add(normalizeIdentifier(columnName)); + } + } + } + return { tables, columns }; +}; + +const validateAssertionDrafts = ( + requirementId: string, + drafts: AnalysisAssertionDraft[], + schema: PhysicalSchemaIndex +): AnalysisContractGroundingFinding[] => { + const findings: AnalysisContractGroundingFinding[] = []; + const tableNames = drafts.flatMap((draft) => [ + ...(draft.sourceTables ?? []), + ...(draft.sqlConstraints ?? []).flatMap((constraint) => constraint.kind === "source" ? [constraint.table] : []) + ]); + const columnNames = drafts.flatMap((draft) => [ + ...(draft.dimensions ?? []), + ...(draft.sqlConstraints ?? []).flatMap(constraintColumns) + ]); + for (const table of new Set(tableNames)) { + if (!schema.tables.has(normalizeIdentifier(table))) { + findings.push({ + requirementId, + code: "CONTRACT_UNKNOWN_TABLE", + message: `Table '${table}' does not exist in the inspected schema.` + }); + } + } + for (const column of new Set(columnNames.filter((value) => value !== "*"))) { + if (!schema.columns.has(normalizeIdentifier(column))) { + findings.push({ + requirementId, + code: "CONTRACT_UNKNOWN_COLUMN", + message: `Column '${column}' does not exist in the inspected schema.` + }); + } + } + return findings; +}; + +const constraintColumns = (constraint: SqlSemanticConstraint): string[] => { + if (constraint.kind === "column" || constraint.kind === "filter" || constraint.kind === "time_range") { + return [constraint.column]; + } + if (constraint.kind === "aggregate") { + return constraint.column ? [constraint.column] : []; + } + return constraint.kind === "group_by" ? constraint.columns : []; +}; + +const normalizeIdentifier = (value: string): string => value.trim().toLocaleLowerCase(); + +const recordValue = (value: unknown, key: string): unknown => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record)[key] + : undefined; + +const recordString = (value: unknown, ...keys: string[]): string | undefined => { + for (const key of keys) { + const field = recordValue(value, key); + if (typeof field === "string" && field.length > 0) { + return field; + } + } + return undefined; +}; diff --git a/packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.test.ts b/packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.test.ts new file mode 100644 index 0000000..e23192b --- /dev/null +++ b/packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.test.ts @@ -0,0 +1,175 @@ +import { describe, expect, it } from "vitest"; + +import { + createAnalysisRequirementExtractionPrompt, + createFallbackAnalysisRequirements, + parseAnalysisRequirementExtractionText +} from "./model-analysis-requirement-extractor.js"; + +describe("analysis requirement extraction", () => { + it("parses fenced JSON and assigns stable requirement ids", () => { + const requirements = parseAnalysisRequirementExtractionText(`\`\`\`json + {"requirements":[ + {"kind":"data_quality","description":"核对利润公式","acceptanceCriteria":["报告误差数量"]}, + {"kind":"metric","description":"计算新增利润","acceptanceCriteria":["精确到分"]} + ]} + \`\`\``); + + expect(requirements).toMatchObject([ + { id: "R1", kind: "data_quality", description: "核对利润公式", required: true, source: "user" }, + { id: "R2", kind: "metric", description: "计算新增利润", required: true, source: "user" } + ]); + }); + + it("deduplicates equivalent descriptions without trusting model ids", () => { + const requirements = parseAnalysisRequirementExtractionText(JSON.stringify({ + requirements: [ + { + id: "invented-1", + kind: "comparison", + description: "Compare holdout cohorts", + acceptanceCriteria: ["show sample sizes"] + }, + { + id: "invented-2", + kind: "comparison", + description: " compare holdout cohorts ", + acceptanceCriteria: ["duplicate"] + } + ] + })); + + expect(requirements).toHaveLength(1); + expect(requirements[0]?.id).toBe("R1"); + }); + + it("normalizes a single acceptance criterion returned as a string", () => { + const requirements = parseAnalysisRequirementExtractionText(JSON.stringify({ + requirements: [{ + kind: "metric", + description: "计算整体转化率", + acceptanceCriteria: "报告分子、分母和比例" + }] + })); + + expect(requirements[0]?.acceptanceCriteria).toEqual(["报告分子、分母和比例"]); + }); + + it("downgrades legacy structured assertions until schema grounding", () => { + const requirements = parseAnalysisRequirementExtractionText(JSON.stringify({ + requirements: [{ + kind: "metric", + description: "计算验证期订单数", + acceptanceCriteria: ["包含完整结束日期"], + assertions: [{ + kind: "metric", + description: "验证期订单数", + sourceTables: ["orders"], + sqlConstraints: [{ + kind: "time_range", + column: "order_date", + start: "2023-07-01", + end: "2023-12-31", + endInclusive: true + }], + resultChecks: [{ kind: "non_empty", required: true }], + claimValues: [{ name: "order_count", field: "order_count", required: true, unit: "orders" }] + }] + }] + })); + + expect(requirements[0]?.assertions).toEqual([expect.objectContaining({ + id: "R1.A1", + requirementId: "R1", + kind: "manual", + sourceTables: [], + dimensions: [], + sqlConstraints: [], + resultChecks: [], + claimValues: [] + })]); + }); + + it("does not trust physical identifiers extracted before schema grounding", () => { + const requirements = parseAnalysisRequirementExtractionText(JSON.stringify({ + requirements: [{ + kind: "metric", + description: "获取2023年订单记录数及日期范围", + acceptanceCriteria: ["报告订单数和日期边界"], + assertions: [{ + kind: "metric", + description: "订单数与日期范围", + sourceTables: ["orders"], + sqlConstraints: [ + { kind: "source", table: "orders" }, + { + kind: "time_range", + column: "order_date", + start: "2023-01-01", + end: "2023-12-31", + endInclusive: true + } + ], + resultChecks: [{ kind: "row_count", required: true, min: 1, max: 1 }], + claimValues: [{ name: "order_count", field: "order_count", required: true }] + }] + }] + })); + + expect(requirements[0]?.assertions).toEqual([ + expect.objectContaining({ + id: "R1.A1", + kind: "manual", + sourceTables: [], + dimensions: [], + sqlConstraints: [], + resultChecks: [], + claimValues: [] + }) + ]); + }); + + it("marks requirements without structured assertions as manual", () => { + const requirements = parseAnalysisRequirementExtractionText(JSON.stringify({ + requirements: [{ + kind: "decision", + description: "判断策略是否合理", + acceptanceCriteria: ["给出结论"] + }] + })); + + expect(requirements[0]?.assertions).toMatchObject([{ + id: "R1.A1", + kind: "manual", + description: "判断策略是否合理" + }]); + }); + + it("rejects unsupported requirement kinds", () => { + expect(() => parseAnalysisRequirementExtractionText(JSON.stringify({ + requirements: [{ kind: "workflow", description: "do work", acceptanceCriteria: [] }] + }))).toThrow(); + }); + + it("creates a conservative evidence-backed fallback requirement", () => { + const requirements = createFallbackAnalysisRequirements("比较两个策略并给出建议"); + + expect(requirements).toMatchObject([{ + id: "R1", + kind: "validation", + description: "比较两个策略并给出建议", + acceptanceCriteria: ["回答请求中的全部可量化问题，并为结论绑定审计证据"] + }]); + }); + + it("builds a tool-free extraction prompt that excludes protocol core requirements", () => { + const prompt = createAnalysisRequirementExtractionPrompt("检查利润公式并比较两个方案"); + + expect(prompt).toContain("检查利润公式并比较两个方案"); + expect(prompt).toContain("不要重复 schema inspection"); + expect(prompt).toContain("不要把输出格式"); + expect(prompt).toContain("只返回一个 JSON 对象"); + expect(prompt).toContain("尚未检查物理 schema"); + expect(prompt).toContain("不要输出 assertions"); + }); +}); diff --git a/packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.ts b/packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.ts new file mode 100644 index 0000000..583de7a --- /dev/null +++ b/packages/agent-runtime/src/protocol/model-analysis-requirement-extractor.ts @@ -0,0 +1,103 @@ +import { Agent } from "@mastra/core/agent"; +import type { ModelProvider } from "@datafoundry/providers"; +import { z } from "zod"; + +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; +import { + ANALYSIS_REQUIREMENT_KINDS, + createUserAnalysisRequirements, + type AnalysisRequirement, + type AnalysisRequirementDraft +} from "./analysis-requirements.js"; +import { analysisAssertionDraftSchema } from "./analysis-contract.js"; + +const requirementDraftSchema = z.object({ + id: z.string().optional(), + kind: z.enum(ANALYSIS_REQUIREMENT_KINDS), + description: z.string().min(1).max(500), + acceptanceCriteria: z.preprocess( + (value) => typeof value === "string" ? [value] : value, + z.array(z.string().min(1).max(300)).max(8) + ), + assertions: z.array(analysisAssertionDraftSchema).max(12).optional() +}).strict(); + +const extractionSchema = z.object({ + requirements: z.array(requirementDraftSchema).min(1).max(16) +}).strict(); + +export type AnalysisRequirementExtractor = (input: { userText: string }) => Promise; + +/** Build the constrained prompt for user-specific analysis requirements. */ +export const createAnalysisRequirementExtractionPrompt = (userText: string): string => [ + "你是数据分析验收条件提取器，不是执行任务的 Agent。", + "把用户请求拆成可由 SQL 结果和审计证据验证的必答要求。", + "当前阶段尚未检查物理 schema。不得猜测或输出任何表名、字段名、SQL 过滤条件或聚合字段。", + "只提取业务目标和验收标准；物理 assertions 会在 schema 与语义解析完成后由独立步骤生成。", + "不要重复 schema inspection、semantic grounding、read-only validation、result validation 或 evidence binding，", + "这些由 Protocol 固定提供。不要加入寒暄、过程说明或可选建议。", + "不要把输出格式、Markdown、文件或下载要求提取为 requirement；它们不属于 SQL 证据验收范围。", + `用户请求: ${userText}`, + `kind 只能是: ${ANALYSIS_REQUIREMENT_KINDS.join(", ")}`, + "只返回一个 JSON 对象，不要 Markdown。字段为 requirements；每项字段只能包含 kind、description 和", + "acceptanceCriteria。acceptanceCriteria 必须是字符串数组，即使只有一项也要使用数组。", + "不要输出 assertions、sourceTables、dimensions、sqlConstraints、resultChecks 或 claimValues。", + "不要生成 id，服务端会分配稳定 ID。" +].join("\n"); + +/** Parse model output and assign stable server-owned requirement IDs. */ +export const parseAnalysisRequirementExtractionText = (text: string): AnalysisRequirement[] => { + const trimmed = text.trim(); + const unfenced = trimmed.startsWith("```") + ? trimmed.replace(/^```(?:json)?\s*/iu, "").replace(/\s*```$/u, "") + : trimmed; + const parsed = extractionSchema.parse(JSON.parse(unfenced) as unknown); + const drafts: AnalysisRequirementDraft[] = parsed.requirements + .filter((requirement) => requirement.kind !== "deliverable") + .map((requirement) => ({ + kind: requirement.kind, + description: requirement.description, + acceptanceCriteria: requirement.acceptanceCriteria + })); + return createUserAnalysisRequirements(drafts); +}; + +/** Create a conservative requirement when model extraction cannot produce valid JSON. */ +export const createFallbackAnalysisRequirements = (userText: string): AnalysisRequirement[] => createUserAnalysisRequirements([{ + kind: "validation", + description: userText.trim().slice(0, 500) || "完成用户请求的数据分析", + acceptanceCriteria: ["回答请求中的全部可量化问题，并为结论绑定审计证据"] +}]); + +/** Create a tool-free structured requirement extractor backed by the configured run model. */ +export const createModelAnalysisRequirementExtractor = ( + provider: Exclude +): AnalysisRequirementExtractor => { + const agent = new Agent({ + id: "analysis-requirement-extractor", + name: "Analysis Requirement Extractor", + instructions: "Extract verification requirements only. Never answer the user's task.", + model: provider.model as never + }); + return async (input) => { + const basePrompt = createAnalysisRequirementExtractionPrompt(input.userText); + for (let attempt = 0; attempt < 2; attempt += 1) { + const retryInstruction = attempt === 0 + ? "" + : "\n上次输出不是合法 JSON。请缩短描述，确保所有字符串闭合，并严格返回紧凑 JSON。"; + const output = await agent.generate(`${basePrompt}${retryInstruction}`, { + maxSteps: AGENT_RUNTIME_LIMITS.modelHelperMaxSteps, + modelSettings: { + maxOutputTokens: AGENT_RUNTIME_LIMITS.requirementExtractorMaxOutputTokens, + temperature: 0 + } + }); + try { + return parseAnalysisRequirementExtractionText(output.text); + } catch { + // Retry once before falling back to a conservative server-owned requirement. + } + } + return createFallbackAnalysisRequirements(input.userText); + }; +}; diff --git a/packages/agent-runtime/src/protocol/model-protocol-classifier.test.ts b/packages/agent-runtime/src/protocol/model-protocol-classifier.test.ts new file mode 100644 index 0000000..df5cae4 --- /dev/null +++ b/packages/agent-runtime/src/protocol/model-protocol-classifier.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { + createProtocolClassificationPrompt, + parseProtocolClassificationText +} from "./model-protocol-classifier.js"; + +describe("createProtocolClassificationPrompt", () => { + it("constrains classification to the router-provided candidates", () => { + const prompt = createProtocolClassificationPrompt({ + candidates: [ + { protocolId: "general-task", protocolVersion: "1" }, + { protocolId: "data-analysis", protocolVersion: "1" } + ], + value: { userText: "比较订单趋势" } + }); + + expect(prompt).toContain("general-task@1"); + expect(prompt).toContain("data-analysis@1"); + expect(prompt).toContain("比较订单趋势"); + expect(prompt).toContain("只能选择候选集合中的协议"); + expect(prompt).toContain('"reasonCodes":["ANALYTIC_INTENT"]'); + }); + + it("strictly parses fenced JSON returned by compatible models", () => { + expect(parseProtocolClassificationText(`\`\`\`json +{"protocolId":"general-task","protocolVersion":"1","confidence":0.9,"reasonCodes":["GENERAL_EXPLANATION"]} +\`\`\``)).toEqual({ + protocolId: "general-task", + protocolVersion: "1", + confidence: 0.9, + reasonCodes: ["GENERAL_EXPLANATION"] + }); + }); +}); diff --git a/packages/agent-runtime/src/protocol/model-protocol-classifier.ts b/packages/agent-runtime/src/protocol/model-protocol-classifier.ts new file mode 100644 index 0000000..cd6d7cb --- /dev/null +++ b/packages/agent-runtime/src/protocol/model-protocol-classifier.ts @@ -0,0 +1,57 @@ +import { Agent } from "@mastra/core/agent"; +import type { ModelProvider } from "@datafoundry/providers"; +import { z } from "zod"; + +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; +import type { ProtocolClassifier, ProtocolIdentity } from "./protocol-router.js"; + +const classificationSchema = z.object({ + protocolId: z.string().min(1), + protocolVersion: z.string().min(1), + confidence: z.number().min(0).max(1), + reasonCodes: z.array(z.string().regex(/^[A-Z][A-Z0-9_]*$/u)).max(4) +}).strict(); + +/** Build the constrained prompt consumed by the protocol-only classifier. */ +export const createProtocolClassificationPrompt = (input: { + candidates: ProtocolIdentity[]; + value: unknown; +}): string => [ + "你是协议路由分类器，不是执行任务的 Agent。", + "只能选择候选集合中的协议，不得发明协议或调用工具。", + "data-analysis 用于需要数据源、schema、SQL、指标、统计或数据结论的任务。", + "general-task 用于日常问答、解释、总结、文件、知识检索和普通协作任务。", + `候选集合: ${input.candidates.map((item) => `${item.protocolId}@${item.protocolVersion}`).join(", ")}`, + `分类输入: ${JSON.stringify(input.value)}`, + "只返回一个 JSON 对象，不要 Markdown。字段为 protocolId、protocolVersion、confidence、reasonCodes。", + '格式示例: {"protocolId":"data-analysis","protocolVersion":"1","confidence":0.91,"reasonCodes":["ANALYTIC_INTENT"]}', + "reasonCodes 只能使用大写英文与下划线。" +].join("\n"); + +/** Parse model text into the strict classifier contract without trusting provider-specific JSON modes. */ +export const parseProtocolClassificationText = (text: string): z.infer => { + const trimmed = text.trim(); + const unfenced = trimmed.startsWith("```") + ? trimmed.replace(/^```(?:json)?\s*/iu, "").replace(/\s*```$/u, "") + : trimmed; + return classificationSchema.parse(JSON.parse(unfenced) as unknown); +}; + +/** Create a tool-free structured classifier backed by the configured run model. */ +export const createModelProtocolClassifier = ( + provider: Exclude +): ProtocolClassifier => { + const agent = new Agent({ + id: "protocol-route-classifier", + name: "Protocol Route Classifier", + instructions: "Classify only. Never answer the user's task.", + model: provider.model as never + }); + return async (input) => { + const output = await agent.generate(createProtocolClassificationPrompt(input), { + maxSteps: AGENT_RUNTIME_LIMITS.modelHelperMaxSteps, + modelSettings: { maxOutputTokens: AGENT_RUNTIME_LIMITS.protocolClassifierMaxOutputTokens, temperature: 0 } + }); + return parseProtocolClassificationText(output.text); + }; +}; diff --git a/packages/agent-runtime/src/protocol/protocol-handoff-coordinator.test.ts b/packages/agent-runtime/src/protocol/protocol-handoff-coordinator.test.ts new file mode 100644 index 0000000..7d10427 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-handoff-coordinator.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; + +import { InMemoryProtocolStateStore } from "./in-memory-protocol-state-store.js"; +import { ProtocolHandoffCoordinator } from "./protocol-handoff-coordinator.js"; +import { ProtocolRegistry } from "./protocol-registry.js"; +import { ProtocolRuntime } from "./protocol-runtime.js"; +import type { AgentProtocolDefinition, ProtocolEvent } from "./types.js"; + +describe("ProtocolHandoffCoordinator", () => { + it("atomically ends the current segment and starts the target segment", () => { + const store = new InMemoryProtocolStateStore(); + const registry = new ProtocolRegistry(); + registry.register(createDefinition("general-task")); + registry.register(createDefinition("data-analysis")); + const currentRuntime = new ProtocolRuntime(createDefinition("general-task"), store); + const current = currentRuntime.start({ + runId: "run-1", + segmentId: "run-1:segment:1", + contextPackageRef: { packageId: "context-1", revision: 3 } + }); + const events: ProtocolEvent[] = []; + const coordinator = new ProtocolHandoffCoordinator(registry, store, { + onEvent: (event) => events.push(event) + }); + + const result = coordinator.handoff({ + runId: "run-1", + segmentId: current.segmentId, + expectedRevision: current.revision, + authorizedProtocolIds: ["general-task", "data-analysis"], + target: { protocolId: "data-analysis", protocolVersion: "1" }, + reasonCodes: ["ANALYTIC_INTENT"], + unresolvedGoals: [] + }); + + expect(store.get("run-1", "run-1:segment:1")).toMatchObject({ + status: "handed_off", + revision: 1 + }); + expect(result.next).toMatchObject({ + protocolId: "data-analysis", + segmentId: "run-1:segment:2", + status: "active", + contextPackageRef: { packageId: "context-1", revision: 3 } + }); + expect(store.get("run-1").segmentId).toBe("run-1:segment:2"); + expect(events.map((event) => event.type)).toEqual([ + "protocol.handoff.proposed", + "protocol.segment.ended", + "protocol.handoff.accepted", + "protocol.segment.started" + ]); + }); +}); + +const createDefinition = (id: string): AgentProtocolDefinition> => ({ + id, + version: "1", + initialPhase: "work", + phases: { work: { allowedActions: [], transitions: [] } }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["WORK_REMAINS"], allowedActions: [] }) +}); diff --git a/packages/agent-runtime/src/protocol/protocol-handoff-coordinator.ts b/packages/agent-runtime/src/protocol/protocol-handoff-coordinator.ts new file mode 100644 index 0000000..26a2739 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-handoff-coordinator.ts @@ -0,0 +1,134 @@ +import { evaluateProtocolHandoff } from "./protocol-handoff.js"; +import type { ProtocolRegistry } from "./protocol-registry.js"; +import type { + ProtocolEvent, + ProtocolRunState, + ProtocolStateStore +} from "./types.js"; +import type { ProtocolIdentity } from "./protocol-router.js"; + +export type ProtocolHandoffCoordinatorOptions = { + onEvent?(event: ProtocolEvent): void; +}; + +export type CoordinateProtocolHandoffInput = { + runId: string; + segmentId: string; + expectedRevision: number; + authorizedProtocolIds: string[]; + target: ProtocolIdentity; + reasonCodes: string[]; + unresolvedGoals: string[]; +}; + +/** Validate a handoff and atomically replace the active protocol segment. */ +export class ProtocolHandoffCoordinator { + constructor( + private readonly registry: ProtocolRegistry, + private readonly store: ProtocolStateStore, + private readonly options: ProtocolHandoffCoordinatorOptions = {} + ) {} + + handoff(input: CoordinateProtocolHandoffInput): { + current: ProtocolRunState; + next: ProtocolRunState; + } { + const current = this.store.get(input.runId, input.segmentId); + this.publish(this.createEvent("protocol.handoff.proposed", current, { + target: input.target, + reasonCodes: input.reasonCodes, + unresolvedGoals: input.unresolvedGoals + })); + const targetDefinition = this.registry.find(input.target.protocolId, input.target.protocolVersion); + if (!targetDefinition) { + this.publish(this.createEvent( + "protocol.handoff.rejected", + current, + { reasonCode: "PROTOCOL_HANDOFF_TARGET_UNAVAILABLE" } + )); + throw new Error("PROTOCOL_HANDOFF_TARGET_UNAVAILABLE"); + } + const decision = evaluateProtocolHandoff({ + authorizedProtocolIds: input.authorizedProtocolIds, + current: { + protocolId: current.protocolId, + protocolVersion: current.protocolVersion, + segmentId: current.segmentId + }, + target: input.target, + reasonCodes: input.reasonCodes, + unresolvedGoals: input.unresolvedGoals + }); + if (decision.status === "rejected") { + this.publish(this.createEvent("protocol.handoff.rejected", current, { reasonCode: decision.reasonCode })); + throw new Error(decision.reasonCode); + } + if (current.status !== "active" && current.status !== "waiting") { + throw new Error(`PROTOCOL_HANDOFF_SOURCE_NOT_ACTIVE:${current.status}`); + } + const ended: ProtocolRunState = { + ...current, + revision: current.revision + 1, + status: "handed_off" + }; + const next: ProtocolRunState = { + protocolId: targetDefinition.id, + protocolVersion: targetDefinition.version, + runId: current.runId, + segmentId: nextSegmentId(current.runId, current.segmentId), + revision: 0, + phase: targetDefinition.initialPhase, + status: "active", + contextPackageRef: current.contextPackageRef, + actions: [], + completionRejections: 0, + domain: targetDefinition.createInitialState({ + contextPackageRef: current.contextPackageRef, + runId: current.runId + }) + }; + const events = [ + this.createEvent("protocol.segment.ended", ended, { status: "handed_off" }), + this.createEvent("protocol.handoff.accepted", next, { + previousSegmentId: current.segmentId, + reasonCodes: decision.reasonCodes + }), + this.createEvent("protocol.segment.started", next, { phase: next.phase }) + ]; + const persisted = this.store.transitionSegment({ + current: ended, + expectedRevision: input.expectedRevision, + next, + events + }); + events.forEach((event) => this.publish(event)); + return persisted; + } + + private createEvent(type: string, state: ProtocolRunState, payload?: unknown): ProtocolEvent { + return { + eventId: `${state.segmentId}:${state.revision}:${type}`, + type, + runId: state.runId, + segmentId: state.segmentId, + protocolId: state.protocolId, + protocolVersion: state.protocolVersion, + revision: state.revision, + ...(payload === undefined ? {} : { payload }) + }; + } + + private publish(event: ProtocolEvent): void { + if (!this.options.onEvent) { + return; + } + this.options.onEvent(event); + this.store.acknowledgeEvent(event); + } +} + +const nextSegmentId = (runId: string, currentSegmentId: string): string => { + const match = currentSegmentId.match(/:segment:(\d+)$/u); + const nextIndex = match ? Number(match[1]) + 1 : 2; + return `${runId}:segment:${nextIndex}`; +}; diff --git a/packages/agent-runtime/src/protocol/protocol-handoff.test.ts b/packages/agent-runtime/src/protocol/protocol-handoff.test.ts new file mode 100644 index 0000000..62149ff --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-handoff.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { evaluateProtocolHandoff } from "./protocol-handoff.js"; + +describe("evaluateProtocolHandoff", () => { + it("rejects a strict-to-general handoff while strict goals remain unresolved", () => { + const decision = evaluateProtocolHandoff({ + authorizedProtocolIds: ["general-task", "data-analysis"], + current: { protocolId: "data-analysis", protocolVersion: "1", segmentId: "segment-1" }, + target: { protocolId: "general-task", protocolVersion: "1" }, + reasonCodes: ["USER_CHANGED_TOPIC"], + unresolvedGoals: ["validate generated SQL"] + }); + + expect(decision).toEqual({ + status: "rejected", + reasonCode: "PROTOCOL_HANDOFF_UNRESOLVED_STRICT_GOALS" + }); + }); + + it("rejects a handoff to an unauthorized protocol", () => { + const decision = evaluateProtocolHandoff({ + authorizedProtocolIds: ["general-task"], + current: { protocolId: "general-task", protocolVersion: "1", segmentId: "segment-1" }, + target: { protocolId: "data-analysis", protocolVersion: "1" }, + reasonCodes: ["ANALYTIC_INTENT"], + unresolvedGoals: [] + }); + + expect(decision).toEqual({ + status: "rejected", + reasonCode: "PROTOCOL_HANDOFF_NOT_AUTHORIZED" + }); + }); +}); diff --git a/packages/agent-runtime/src/protocol/protocol-handoff.ts b/packages/agent-runtime/src/protocol/protocol-handoff.ts new file mode 100644 index 0000000..b5bfe99 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-handoff.ts @@ -0,0 +1,34 @@ +import type { ProtocolIdentity } from "./protocol-router.js"; + +export type ProtocolSegmentIdentity = ProtocolIdentity & { + segmentId: string; +}; + +export type ProtocolHandoffInput = { + authorizedProtocolIds: string[]; + current: ProtocolSegmentIdentity; + target: ProtocolIdentity; + reasonCodes: string[]; + strictProtocolIds?: string[]; + unresolvedGoals: string[]; +}; + +export type ProtocolHandoffDecision = + | { status: "accepted"; reasonCodes: string[]; target: ProtocolIdentity } + | { status: "rejected"; reasonCode: string }; + +/** Evaluate whether a proposed protocol handoff may create a new segment. */ +export const evaluateProtocolHandoff = (input: ProtocolHandoffInput): ProtocolHandoffDecision => { + if (!input.authorizedProtocolIds.includes(input.target.protocolId)) { + return { status: "rejected", reasonCode: "PROTOCOL_HANDOFF_NOT_AUTHORIZED" }; + } + const strictProtocolIds = new Set(input.strictProtocolIds ?? ["data-analysis"]); + if ( + strictProtocolIds.has(input.current.protocolId) + && !strictProtocolIds.has(input.target.protocolId) + && input.unresolvedGoals.length > 0 + ) { + return { status: "rejected", reasonCode: "PROTOCOL_HANDOFF_UNRESOLVED_STRICT_GOALS" }; + } + return { status: "accepted", reasonCodes: input.reasonCodes, target: input.target }; +}; diff --git a/packages/agent-runtime/src/protocol/protocol-public-api.test.ts b/packages/agent-runtime/src/protocol/protocol-public-api.test.ts new file mode 100644 index 0000000..4a146f3 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-public-api.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { + ActionRouter, + CapabilityRegistry, + DataLinkSemanticProvider, + InMemoryProtocolStateStore, + ProtocolRegistry, + ProtocolHandoffCoordinator, + ProtocolRouter, + ProtocolRuntime, + LocalSemanticProvider, + SemanticProviderChain, + evaluateProtocolHandoff, + createToolCapabilityPlugin, + validateProtocolDefinition +} from "../testing.js"; + +describe("protocol testing exports", () => { + it("exposes the protocol kernel through the package testing surface", () => { + expect(ProtocolRegistry).toBeTypeOf("function"); + expect(ProtocolRouter).toBeTypeOf("function"); + expect(ProtocolRuntime).toBeTypeOf("function"); + expect(ProtocolHandoffCoordinator).toBeTypeOf("function"); + expect(InMemoryProtocolStateStore).toBeTypeOf("function"); + expect(validateProtocolDefinition).toBeTypeOf("function"); + expect(evaluateProtocolHandoff).toBeTypeOf("function"); + expect(CapabilityRegistry).toBeTypeOf("function"); + expect(ActionRouter).toBeTypeOf("function"); + expect(createToolCapabilityPlugin).toBeTypeOf("function"); + expect(DataLinkSemanticProvider).toBeTypeOf("function"); + expect(LocalSemanticProvider).toBeTypeOf("function"); + expect(SemanticProviderChain).toBeTypeOf("function"); + }); +}); diff --git a/packages/agent-runtime/src/protocol/protocol-registry.ts b/packages/agent-runtime/src/protocol/protocol-registry.ts new file mode 100644 index 0000000..32d6a89 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-registry.ts @@ -0,0 +1,27 @@ +import { validateProtocolDefinition } from "./definition-validator.js"; +import type { AgentProtocolDefinition } from "./types.js"; + +export type RegisteredProtocolDefinition = AgentProtocolDefinition; + +export class ProtocolRegistry { + private readonly definitions = new Map(); + + register(definition: RegisteredProtocolDefinition): void { + validateProtocolDefinition(definition); + const key = protocolDefinitionKey(definition.id, definition.version); + if (this.definitions.has(key)) { + throw new Error(`PROTOCOL_ALREADY_REGISTERED:${key}`); + } + this.definitions.set(key, definition); + } + + list(): RegisteredProtocolDefinition[] { + return [...this.definitions.values()]; + } + + find(id: string, version: string): RegisteredProtocolDefinition | undefined { + return this.definitions.get(protocolDefinitionKey(id, version)); + } +} + +export const protocolDefinitionKey = (id: string, version: string): string => `${id}@${version}`; diff --git a/packages/agent-runtime/src/protocol/protocol-router.test.ts b/packages/agent-runtime/src/protocol/protocol-router.test.ts new file mode 100644 index 0000000..670ef3c --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-router.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from "vitest"; + +import { ProtocolRegistry } from "./protocol-registry.js"; +import { ProtocolRouter } from "./protocol-router.js"; +import type { AgentProtocolDefinition } from "./types.js"; + +describe("ProtocolRouter", () => { + it("rejects a run when no authorized protocol can be resolved", async () => { + const router = new ProtocolRouter(createRegistry()); + + await expect(router.route({ authorizedProtocolIds: [] })) + .rejects.toThrow("PROTOCOL_NOT_RESOLVED"); + }); + + it("rejects an unauthorized explicit protocol", async () => { + const router = new ProtocolRouter(createRegistry()); + + await expect(router.route({ + authorizedProtocolIds: ["general-task"], + explicit: { protocolId: "data-analysis", protocolVersion: "1" } + })).rejects.toThrow("PROTOCOL_NOT_AUTHORIZED:data-analysis@1"); + }); + + it("honors an explicitly selected authorized protocol", async () => { + const registry = createRegistry(); + const router = new ProtocolRouter(registry); + + const result = await router.route({ + authorizedProtocolIds: ["general-task", "data-analysis"], + explicit: { protocolId: "data-analysis", protocolVersion: "1" } + }); + + expect(result.definition.id).toBe("data-analysis"); + expect(result.source).toBe("explicit"); + }); + + it("selects the highest-priority deterministic route", async () => { + const router = new ProtocolRouter(createRegistry()); + + const result = await router.route({ + authorizedProtocolIds: ["general-task", "data-analysis"], + deterministicCandidates: [ + { protocolId: "general-task", protocolVersion: "1", priority: 10, reasonCode: "DEFAULT_TASK" }, + { protocolId: "data-analysis", protocolVersion: "1", priority: 100, reasonCode: "ANALYTIC_INTENT" } + ] + }); + + expect(result.definition.id).toBe("data-analysis"); + expect(result.source).toBe("deterministic"); + expect(result.reasonCodes).toEqual(["ANALYTIC_INTENT"]); + }); + + it("uses a constrained classifier when deterministic rules do not resolve", async () => { + const router = new ProtocolRouter(createRegistry(), { + classifier: async ({ candidates }) => { + expect(candidates.map((candidate) => candidate.protocolId).sort()) + .toEqual(["data-analysis", "general-task"]); + return { + protocolId: "data-analysis", + protocolVersion: "1", + confidence: 0.92, + reasonCodes: ["ANALYTIC_INTENT"] + }; + } + }); + + const result = await router.route({ + authorizedProtocolIds: ["general-task", "data-analysis"], + classificationInput: { userText: "按月分析销售额" } + }); + + expect(result.definition.id).toBe("data-analysis"); + expect(result.source).toBe("classifier"); + }); + + it("uses the authorized general protocol when classifier confidence is low", async () => { + const router = new ProtocolRouter(createRegistry(), { + classifier: async () => ({ + protocolId: "data-analysis", + protocolVersion: "1", + confidence: 0.4, + reasonCodes: ["WEAK_ANALYTIC_INTENT"] + }) + }); + + const result = await router.route({ + authorizedProtocolIds: ["general-task", "data-analysis"], + classificationInput: { userText: "帮我看看" } + }); + + expect(result.definition.id).toBe("general-task"); + expect(result.source).toBe("default"); + expect(result.warnings).toEqual(["PROTOCOL_CLASSIFICATION_LOW_CONFIDENCE"]); + }); + + it("uses the authorized general protocol when classification fails transiently", async () => { + const router = new ProtocolRouter(createRegistry(), { + classifier: async () => { + throw new Error("MODEL_TEMPORARILY_UNAVAILABLE"); + } + }); + + const result = await router.route({ + authorizedProtocolIds: ["general-task", "data-analysis"], + classificationInput: { userText: "介绍一下这个项目" } + }); + + expect(result.definition.id).toBe("general-task"); + expect(result.warnings).toEqual(["PROTOCOL_CLASSIFICATION_FAILED"]); + }); + + it("rejects equally ranked deterministic routes without a classifier", async () => { + const router = new ProtocolRouter(createRegistry()); + + await expect(router.route({ + authorizedProtocolIds: ["general-task", "data-analysis"], + deterministicCandidates: [ + { protocolId: "general-task", protocolVersion: "1", priority: 50, reasonCode: "SKILL_ROUTE" }, + { protocolId: "data-analysis", protocolVersion: "1", priority: 50, reasonCode: "TASK_ROUTE" } + ] + })).rejects.toThrow("PROTOCOL_AMBIGUOUS:data-analysis@1,general-task@1"); + }); + + it("rejects duplicate protocol registrations", () => { + const registry = new ProtocolRegistry(); + registry.register(createDefinition("general-task")); + + expect(() => registry.register(createDefinition("general-task"))) + .toThrow("PROTOCOL_ALREADY_REGISTERED:general-task@1"); + }); +}); + +const createRegistry = (): ProtocolRegistry => { + const registry = new ProtocolRegistry(); + registry.register(createDefinition("general-task")); + registry.register(createDefinition("data-analysis")); + return registry; +}; + +const createDefinition = (id: string): AgentProtocolDefinition => ({ + id, + version: "1", + initialPhase: "active", + phases: { active: { allowedActions: [], transitions: [] } }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) +}); diff --git a/packages/agent-runtime/src/protocol/protocol-router.ts b/packages/agent-runtime/src/protocol/protocol-router.ts new file mode 100644 index 0000000..2d2445e --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-router.ts @@ -0,0 +1,133 @@ +import type { ProtocolRegistry, RegisteredProtocolDefinition } from "./protocol-registry.js"; + +export type ProtocolIdentity = { + protocolId: string; + protocolVersion: string; +}; + +export type ProtocolRouteSource = "explicit" | "deterministic" | "classifier" | "default"; + +export type ProtocolRouteInput = { + authorizedProtocolIds: string[]; + classificationInput?: unknown; + deterministicCandidates?: Array; + explicit?: ProtocolIdentity; +}; + +export type ProtocolRouteClassification = ProtocolIdentity & { + confidence: number; + reasonCodes: string[]; +}; + +export type ProtocolClassifier = (input: { + candidates: ProtocolIdentity[]; + value: unknown; +}) => Promise; + +export type ProtocolRouterOptions = { + classifier?: ProtocolClassifier; + confidenceThreshold?: number; + defaultProtocol?: ProtocolIdentity; +}; + +export type ProtocolRouteResult = { + definition: RegisteredProtocolDefinition; + reasonCodes: string[]; + source: ProtocolRouteSource; + warnings: string[]; +}; + +export class ProtocolRouter { + constructor( + private readonly registry: ProtocolRegistry, + private readonly options: ProtocolRouterOptions = {} + ) {} + + async route(input: ProtocolRouteInput): Promise { + let defaultWarning: string | undefined; + if (input.explicit) { + if (!input.authorizedProtocolIds.includes(input.explicit.protocolId)) { + throw new Error(`PROTOCOL_NOT_AUTHORIZED:${input.explicit.protocolId}@${input.explicit.protocolVersion}`); + } + const definition = this.registry.find(input.explicit.protocolId, input.explicit.protocolVersion); + if (!definition) { + throw new Error(`PROTOCOL_NOT_REGISTERED:${input.explicit.protocolId}@${input.explicit.protocolVersion}`); + } + return { definition, reasonCodes: ["USER_EXPLICIT"], source: "explicit", warnings: [] }; + } + const deterministicCandidates = (input.deterministicCandidates ?? []) + .filter((candidate) => input.authorizedProtocolIds.includes(candidate.protocolId)) + .map((candidate) => ({ + candidate, + definition: this.registry.find(candidate.protocolId, candidate.protocolVersion) + })) + .filter((entry): entry is typeof entry & { definition: RegisteredProtocolDefinition } => Boolean(entry.definition)) + .sort((left, right) => right.candidate.priority - left.candidate.priority); + const selected = deterministicCandidates[0]; + const equallyRanked = selected + ? deterministicCandidates.filter((entry) => entry.candidate.priority === selected.candidate.priority) + : []; + if (selected && equallyRanked.length === 1) { + return { + definition: selected.definition, + reasonCodes: [selected.candidate.reasonCode], + source: "deterministic", + warnings: [] + }; + } + if (equallyRanked.length > 1 && !this.options.classifier) { + const keys = equallyRanked + .map((entry) => `${entry.definition.id}@${entry.definition.version}`) + .sort(); + throw new Error(`PROTOCOL_AMBIGUOUS:${keys.join(",")}`); + } + if (this.options.classifier) { + const candidates = this.registry.list() + .filter((definition) => input.authorizedProtocolIds.includes(definition.id)) + .map((definition) => ({ protocolId: definition.id, protocolVersion: definition.version })); + let classification: ProtocolRouteClassification | undefined; + try { + classification = await this.options.classifier({ candidates, value: input.classificationInput }); + } catch { + defaultWarning = "PROTOCOL_CLASSIFICATION_FAILED"; + } + if (classification) { + const definition = candidates.some((candidate) => + candidate.protocolId === classification.protocolId + && candidate.protocolVersion === classification.protocolVersion) + ? this.registry.find(classification.protocolId, classification.protocolVersion) + : undefined; + if (!definition) { + throw new Error( + `PROTOCOL_CLASSIFIER_INVALID_SELECTION:${classification.protocolId}@${classification.protocolVersion}` + ); + } + if (classification.confidence >= (this.options.confidenceThreshold ?? 0.75)) { + return { + definition, + reasonCodes: classification.reasonCodes, + source: "classifier", + warnings: [] + }; + } + defaultWarning = "PROTOCOL_CLASSIFICATION_LOW_CONFIDENCE"; + } + } + const defaultProtocol = this.options.defaultProtocol ?? { + protocolId: "general-task", + protocolVersion: "1" + }; + if (input.authorizedProtocolIds.includes(defaultProtocol.protocolId)) { + const definition = this.registry.find(defaultProtocol.protocolId, defaultProtocol.protocolVersion); + if (definition) { + return { + definition, + reasonCodes: ["GENERAL_TASK_DEFAULT"], + source: "default", + warnings: defaultWarning ? [defaultWarning] : [] + }; + } + } + throw new Error("PROTOCOL_NOT_RESOLVED"); + } +} diff --git a/packages/agent-runtime/src/protocol/protocol-runtime.test.ts b/packages/agent-runtime/src/protocol/protocol-runtime.test.ts new file mode 100644 index 0000000..8b4c845 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-runtime.test.ts @@ -0,0 +1,350 @@ +import { describe, expect, it } from "vitest"; + +import { InMemoryProtocolStateStore } from "./in-memory-protocol-state-store.js"; +import { ProtocolRuntime } from "./protocol-runtime.js"; +import type { AgentProtocolDefinition, ContextPackageRef } from "./types.js"; + +const contextPackageRef: ContextPackageRef = { packageId: "context-1", revision: 3 }; + +describe("ProtocolRuntime", () => { + it("starts a run in the protocol initial phase with its ContextPackage reference", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + + const state = runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + expect(state).toMatchObject({ + protocolId: "test/protocol", + protocolVersion: "1", + runId: "run-1", + segmentId: "segment-1", + revision: 0, + phase: "inspect", + status: "active", + contextPackageRef, + actions: [], + domain: { inspected: false } + }); + }); + + it("rejects an action that is not allowed in the current phase", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + expect(() => runtime.assertActionAllowed({ + runId: "run-1", + actionName: "data.query", + actionInput: {} + })).toThrow("ACTION_NOT_ALLOWED_IN_PHASE:inspect:data.query"); + }); + + it("rejects an action when a phase guard denies it", () => { + const definition = createDefinition(); + const inspectPhase = definition.phases.inspect; + if (!inspectPhase) { + throw new Error("TEST_PHASE_REQUIRED"); + } + inspectPhase.actionGuards = { + "data.inspect": [() => ({ allowed: false, reasonCode: "DATASOURCE_REQUIRED" })] + }; + const runtime = new ProtocolRuntime(definition, new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + expect(() => runtime.assertActionAllowed({ + runId: "run-1", + actionName: "data.inspect", + actionInput: {} + })).toThrow("PROTOCOL_GUARD_REJECTED:DATASOURCE_REQUIRED:data.inspect"); + }); + + it("records a successful action and transitions using the reduced domain state", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + const outputContextPackageRef = { packageId: "context-1", revision: 4 }; + beginInspect(runtime, "action-1"); + + const state = runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + reduceDomain: () => ({ inspected: true }), + outputContextPackageRef + }); + + expect(state).toMatchObject({ + revision: 2, + phase: "query", + domain: { inspected: true }, + contextPackageRef: outputContextPackageRef, + actions: [{ + actionId: "action-1", + actionName: "data.inspect", + status: "succeeded", + inputContextPackageRef: contextPackageRef, + outputContextPackageRef + }] + }); + }); + + it("commits a late result admitted in an earlier phase without regressing the current phase", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + beginInspect(runtime, "action-1"); + beginInspect(runtime, "action-2"); + runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + reduceDomain: () => ({ inspected: true }), + outputContextPackageRef: { packageId: "context-1", revision: 4 } + }); + + const state = runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-2", + actionName: "data.inspect", + reduceDomain: (domain) => domain, + outputContextPackageRef: { packageId: "context-1", revision: 5 } + }); + + expect(state).toMatchObject({ + revision: 4, + phase: "query", + domain: { inspected: true }, + actions: [ + { actionId: "action-1", status: "succeeded" }, + { actionId: "action-2", status: "succeeded" } + ] + }); + }); + + it("keeps the current ContextPackage reference when a result has the same revision", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + const current = { ...contextPackageRef, eventId: "current-event" }; + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef: current }); + beginInspect(runtime, "action-1"); + + const state = runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + outputContextPackageRef: { ...contextPackageRef, eventId: "candidate-event" } + }); + + expect(state.contextPackageRef).toEqual(current); + }); + + it("rejects a result from a different ContextPackage lineage", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + beginInspect(runtime, "action-1"); + + expect(() => runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + outputContextPackageRef: { packageId: "context-stale", revision: contextPackageRef.revision } + })).toThrow("PROTOCOL_CONTEXT_PACKAGE_MISMATCH:context-1:context-stale"); + }); + + it("rejects a run whose ContextPackage reference is not resolvable", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + + expect(() => runtime.start({ + runId: "run-1", + segmentId: "segment-1", + contextPackageRef: { packageId: "", revision: -1 } + })).toThrow("PROTOCOL_CONTEXT_REF_INVALID"); + }); + + it("rejects a well-formed ContextPackage reference missing from durable storage", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore(), { + contextPackageExists: () => false + }); + + expect(() => runtime.start({ + runId: "run-1", + segmentId: "segment-1", + contextPackageRef + })).toThrow("PROTOCOL_CONTEXT_REF_NOT_FOUND:context-1@3"); + }); + + it("records a terminal completion decision against the evaluated ContextPackage", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + const outputContextPackageRef = { packageId: "context-1", revision: 4 }; + beginInspect(runtime, "action-1"); + runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + reduceDomain: () => ({ inspected: true }), + outputContextPackageRef + }); + + const state = runtime.proposeCompletion({ + runId: "run-1", + segmentId: "segment-1", + expectedRevision: 2 + }); + + expect(state.status).toBe("terminal"); + expect(state.terminalDecision).toEqual({ + status: "completed", + evaluatedContextPackageRef: outputContextPackageRef, + evidenceRefs: [] + }); + }); + + it("returns partial instead of completed when the completion rejection budget is exhausted", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore(), { + maxCompletionRejections: 1 + }); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + const state = runtime.proposeCompletion({ + runId: "run-1", + segmentId: "segment-1", + expectedRevision: 0 + }); + + expect(state.status).toBe("terminal"); + expect(state.terminalDecision).toEqual({ + status: "partial", + evaluatedContextPackageRef: contextPackageRef, + missing: ["not done"], + evidenceRefs: [] + }); + }); + + it("emits ordered events for run start and initial phase entry", () => { + const events: Array<{ type: string; revision: number }> = []; + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore(), { + onEvent: (event) => events.push({ type: event.type, revision: event.revision }) + }); + + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + expect(events).toEqual([ + { type: "protocol.run.started", revision: 0 }, + { type: "protocol.phase.entered", revision: 0 } + ]); + }); + + it("emits action, transition, and completion events in causal order", () => { + const events: string[] = []; + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore(), { + onEvent: (event) => events.push(event.type) + }); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + events.length = 0; + beginInspect(runtime, "action-1"); + runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + reduceDomain: () => ({ inspected: true }), + outputContextPackageRef: { packageId: "context-1", revision: 4 } + }); + runtime.proposeCompletion({ runId: "run-1", segmentId: "segment-1", expectedRevision: 2 }); + + expect(events).toEqual([ + "protocol.action.requested", + "protocol.action.started", + "protocol.action.succeeded", + "protocol.state.updated", + "protocol.phase.entered", + "protocol.completion.proposed", + "protocol.run.completed" + ]); + }); + + it("rejects actions after the action budget is exhausted", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore(), { maxActions: 1 }); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + beginInspect(runtime, "action-1"); + runtime.recordActionSuccess({ + runId: "run-1", + segmentId: "segment-1", + actionId: "action-1", + actionName: "data.inspect", + reduceDomain: () => ({ inspected: true }), + outputContextPackageRef: { packageId: "context-1", revision: 4 } + }); + + expect(() => runtime.assertActionAllowed({ + runId: "run-1", + actionName: "data.query", + actionInput: {} + })).toThrow("PROTOCOL_ACTION_BUDGET_EXHAUSTED:1"); + }); + + it("returns partial when the time budget is exhausted", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore(), { + deadlineMs: 1000, + now: () => 1000 + }); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + const state = runtime.proposeCompletion({ + runId: "run-1", + segmentId: "segment-1", + expectedRevision: 0 + }); + + expect(state.terminalDecision).toMatchObject({ + status: "partial", + missing: ["TIME_BUDGET_EXHAUSTED"] + }); + }); + + it("returns partial when the model ends with unmet completion requirements", () => { + const runtime = new ProtocolRuntime(createDefinition(), new InMemoryProtocolStateStore()); + runtime.start({ runId: "run-1", segmentId: "segment-1", contextPackageRef }); + + const state = runtime.proposeCompletion({ + runId: "run-1", + segmentId: "segment-1", + expectedRevision: 0, + forceTerminal: true + }); + + expect(state.terminalDecision).toMatchObject({ + status: "partial", + missing: ["not done"] + }); + }); +}); + +const createDefinition = (): AgentProtocolDefinition<{ inspected: boolean }> => ({ + id: "test/protocol", + version: "1", + initialPhase: "inspect", + phases: { + inspect: { + allowedActions: ["data.inspect"], + transitions: [{ targetPhase: "query", when: ({ state }) => state.inspected }] + }, + query: { allowedActions: ["data.query"], transitions: [] } + }, + createInitialState: () => ({ inspected: false }), + completionPolicy: ({ contextPackageRef, state }) => state.inspected + ? { status: "completed", evaluatedContextPackageRef: contextPackageRef, evidenceRefs: [] } + : { status: "continue", reasons: ["not done"], allowedActions: ["data.inspect"] } +}); + +const beginInspect = (runtime: ProtocolRuntime<{ inspected: boolean }>, actionId: string): void => { + runtime.beginAction({ + runId: "run-1", + segmentId: "segment-1", + actionId, + actionName: "data.inspect", + actionInput: {} + }); +}; diff --git a/packages/agent-runtime/src/protocol/protocol-runtime.ts b/packages/agent-runtime/src/protocol/protocol-runtime.ts new file mode 100644 index 0000000..610857f --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocol-runtime.ts @@ -0,0 +1,430 @@ +import { validateProtocolDefinition } from "./definition-validator.js"; +import type { + AgentProtocolDefinition, + ContextPackageRef, + ProtocolEvent, + ProtocolRunState, + ProtocolStateStore +} from "./types.js"; +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; + +export type ProtocolRuntimeOptions = { + contextPackageExists?(reference: ContextPackageRef): boolean; + deadlineMs?: number; + maxActions?: number; + maxCommitRetries?: number; + maxCompletionRejections?: number; + now?(): number; + onEvent?(event: ProtocolEvent): unknown; + startEvents?: Array<{ type: string; payload?: unknown }>; +}; + +export class ProtocolRuntime { + constructor( + private readonly definition: AgentProtocolDefinition, + private readonly stateStore: ProtocolStateStore, + private readonly options: ProtocolRuntimeOptions = {} + ) { + validateProtocolDefinition(definition); + } + + start(input: { + runId: string; + segmentId: string; + contextPackageRef: ContextPackageRef; + }): ProtocolRunState { + this.assertContextPackageRef(input.contextPackageRef); + const initialState: ProtocolRunState = { + protocolId: this.definition.id, + protocolVersion: this.definition.version, + runId: input.runId, + segmentId: input.segmentId, + revision: 0, + phase: this.definition.initialPhase, + status: "active", + contextPackageRef: input.contextPackageRef, + actions: [], + completionRejections: 0, + domain: this.definition.createInitialState(input) + }; + const events = [ + ...(this.options.startEvents ?? []).map((event) => + this.createEvent(event.type, initialState, event.payload)), + this.createEvent("protocol.run.started", initialState), + this.createEvent("protocol.phase.entered", initialState, { phase: initialState.phase }) + ]; + const state = this.stateStore.create(initialState, events); + events.forEach((event) => this.publish(event)); + return state; + } + + assertActionAllowed(input: { + runId: string; + actionName: string; + actionInput: unknown; + }): ProtocolRunState { + const state = this.stateStore.get(input.runId); + this.assertActionAllowedInState(state, input.actionName, input.actionInput); + return state; + } + + private assertActionAllowedInState( + state: ProtocolRunState, + actionName: string, + actionInput: unknown + ): void { + const maxActions = this.options.maxActions ?? AGENT_RUNTIME_LIMITS.protocolDefaultMaxActions; + if (state.actions.length >= maxActions) { + throw new Error(`PROTOCOL_ACTION_BUDGET_EXHAUSTED:${maxActions}`); + } + const phase = this.definition.phases[state.phase]; + if (!phase?.allowedActions.includes(actionName)) { + throw new Error(`ACTION_NOT_ALLOWED_IN_PHASE:${state.phase}:${actionName}`); + } + for (const guard of phase.actionGuards?.[actionName] ?? []) { + const result = guard({ + actionInput, + actionName, + state: state.domain + }); + if (!result.allowed) { + throw new Error(`PROTOCOL_GUARD_REJECTED:${result.reasonCode}:${actionName}`); + } + } + } + + getState(runId: string, segmentId?: string): ProtocolRunState { + return this.stateStore.get(runId, segmentId); + } + + beginAction(input: { + runId: string; + segmentId: string; + actionId: string; + actionName: string; + actionInput: unknown; + }): ProtocolRunState { + return this.commitWithRetry("begin_action", input.runId, input.segmentId, (current) => { + this.assertActionAllowedInState(current, input.actionName, input.actionInput); + const next: ProtocolRunState = { + ...current, + revision: current.revision + 1, + actions: [...current.actions, { + actionId: input.actionId, + actionName: input.actionName, + status: "requested", + inputContextPackageRef: current.contextPackageRef + }] + }; + return { + next, + events: [ + this.createEvent("protocol.action.requested", next, { + actionId: input.actionId, + actionName: input.actionName + }), + this.createEvent("protocol.action.started", next, { + actionId: input.actionId, + actionName: input.actionName + }) + ] + }; + }); + } + + recordActionRejection(input: { + runId: string; + segmentId: string; + actionId: string; + actionName: string; + reasonCode: string; + }): ProtocolRunState { + return this.commitWithRetry("reject_action", input.runId, input.segmentId, (current) => { + const next: ProtocolRunState = { + ...current, + revision: current.revision + 1, + actions: [...current.actions, { + actionId: input.actionId, + actionName: input.actionName, + status: "rejected", + inputContextPackageRef: current.contextPackageRef, + reasonCode: input.reasonCode + }] + }; + return { + next, + events: [ + this.createEvent("protocol.action.requested", next, { + actionId: input.actionId, + actionName: input.actionName + }), + this.createEvent("protocol.action.rejected", next, { + actionId: input.actionId, + actionName: input.actionName, + reasonCode: input.reasonCode + }) + ] + }; + }); + } + + restore(runId: string, segmentId: string): ProtocolRunState { + const state = this.stateStore.get(runId, segmentId); + if (state.protocolId !== this.definition.id || state.protocolVersion !== this.definition.version) { + throw new Error( + `PROTOCOL_RESTORE_DEFINITION_MISMATCH:${state.protocolId}@${state.protocolVersion}:` + + `${this.definition.id}@${this.definition.version}` + ); + } + this.assertContextPackageRef(state.contextPackageRef); + return state; + } + + recordActionSuccess(input: { + runId: string; + segmentId: string; + actionId: string; + actionName: string; + reduceDomain?(domain: TDomainState): TDomainState; + eventResult?: unknown; + outputContextPackageRef: ContextPackageRef; + }): ProtocolRunState { + this.assertContextPackageRef(input.outputContextPackageRef); + return this.commitWithRetry("complete_action", input.runId, input.segmentId, (current) => { + const action = requireRequestedAction(current, input.actionId, input.actionName); + const domain = input.reduceDomain?.(current.domain) ?? current.domain; + const phase = this.definition.phases[current.phase]; + const transitions = phase?.allowedActions.includes(input.actionName) + ? phase.transitions.filter((transition) => transition.when({ actionName: input.actionName, state: domain })) + : []; + if (transitions.length > 1) { + throw new Error(`PROTOCOL_TRANSITION_AMBIGUOUS:${current.phase}:${input.actionName}`); + } + const next: ProtocolRunState = { + ...current, + revision: current.revision + 1, + phase: transitions[0]?.targetPhase ?? current.phase, + contextPackageRef: latestContextPackageRef(current.contextPackageRef, input.outputContextPackageRef), + actions: updateActionRecord(current.actions, input.actionId, { + actionId: input.actionId, + actionName: input.actionName, + status: "succeeded", + inputContextPackageRef: action.inputContextPackageRef, + outputContextPackageRef: input.outputContextPackageRef + }), + domain + }; + const events = [this.createEvent("protocol.action.succeeded", next, { + actionId: input.actionId, + actionName: input.actionName, + ...(input.eventResult === undefined ? {} : { result: input.eventResult }) + }), this.createEvent("protocol.state.updated", next)]; + if (next.phase !== current.phase) { + events.push(this.createEvent("protocol.phase.entered", next, { phase: next.phase })); + } + return { next, events }; + }); + } + + recordActionFailure(input: { + runId: string; + segmentId: string; + actionId: string; + actionName: string; + reasonCode: string; + }): ProtocolRunState { + return this.commitWithRetry("fail_action", input.runId, input.segmentId, (current) => { + const action = requireRequestedAction(current, input.actionId, input.actionName); + const next: ProtocolRunState = { + ...current, + revision: current.revision + 1, + actions: updateActionRecord(current.actions, input.actionId, { + actionId: input.actionId, + actionName: input.actionName, + status: "failed", + inputContextPackageRef: action.inputContextPackageRef, + reasonCode: input.reasonCode + }) + }; + return { + next, + events: [ + this.createEvent("protocol.action.failed", next, { + actionId: input.actionId, + actionName: input.actionName, + reasonCode: input.reasonCode + }), + this.createEvent("protocol.state.updated", next) + ] + }; + }); + } + + proposeCompletion(input: { + runId: string; + segmentId: string; + expectedRevision: number; + forceTerminal?: boolean; + }): ProtocolRunState { + const current = this.stateStore.get(input.runId, input.segmentId); + const decision = this.definition.completionPolicy({ + contextPackageRef: current.contextPackageRef, + state: current.domain + }); + const rejectionCount = current.completionRejections + (decision.status === "continue" ? 1 : 0); + const budgetExhausted = decision.status === "continue" + && rejectionCount >= (this.options.maxCompletionRejections + ?? AGENT_RUNTIME_LIMITS.protocolMaxCompletionRejections); + const timeBudgetExhausted = this.options.deadlineMs !== undefined + && (this.options.now?.() ?? Date.now()) >= this.options.deadlineMs; + const terminalDecision = timeBudgetExhausted + ? { + status: "partial" as const, + evaluatedContextPackageRef: current.contextPackageRef, + missing: ["TIME_BUDGET_EXHAUSTED"], + evidenceRefs: [] + } + : input.forceTerminal && decision.status === "continue" + ? { + status: "partial" as const, + evaluatedContextPackageRef: current.contextPackageRef, + missing: decision.reasons, + evidenceRefs: [] + } + : budgetExhausted + ? { + status: "partial" as const, + evaluatedContextPackageRef: current.contextPackageRef, + missing: decision.reasons, + evidenceRefs: [] + } + : decision; + const next: ProtocolRunState = terminalDecision.status === "continue" + ? { ...current, revision: current.revision + 1, completionRejections: rejectionCount } + : { + ...current, + revision: current.revision + 1, + status: "terminal", + completionRejections: rejectionCount, + terminalDecision + }; + const events = [this.createEvent("protocol.completion.proposed", next, { decision: terminalDecision })]; + if (terminalDecision.status === "continue") { + events.push(this.createEvent("protocol.completion.rejected", next, { decision: terminalDecision })); + } else { + events.push(this.createEvent(`protocol.run.${terminalDecision.status}`, next, { decision: terminalDecision })); + } + const persisted = this.stateStore.compareAndSet(next, input.expectedRevision, events); + events.forEach((event) => this.publish(event)); + return persisted; + } + + private commitWithRetry( + operation: string, + runId: string, + segmentId: string, + build: (current: ProtocolRunState) => { + next: ProtocolRunState; + events: ProtocolEvent[]; + } + ): ProtocolRunState { + const maxAttempts = Math.max(1, (this.options.maxCommitRetries + ?? AGENT_RUNTIME_LIMITS.protocolMaxCommitRetries) + 1); + let lastConflict: unknown; + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const current = this.stateStore.get(runId, segmentId); + const { next, events } = build(current); + try { + const persisted = this.stateStore.compareAndSet(next, current.revision, events); + events.forEach((event) => this.publish(event)); + return persisted; + } catch (error) { + if (!isProtocolRevisionConflict(error)) { + throw error; + } + lastConflict = error; + } + } + const detail = lastConflict instanceof Error ? lastConflict.message : "PROTOCOL_REVISION_CONFLICT"; + throw new Error(`PROTOCOL_COMMIT_CONTENTION:${operation}:${runId}:${segmentId}:${maxAttempts}:${detail}`); + } + + private createEvent( + type: string, + state: ProtocolRunState, + payload?: unknown + ): ProtocolEvent { + return { + eventId: `${state.segmentId}:${state.revision}:${type}`, + type, + runId: state.runId, + segmentId: state.segmentId, + protocolId: state.protocolId, + protocolVersion: state.protocolVersion, + revision: state.revision, + ...(payload === undefined ? {} : { payload }) + }; + } + + private publish(event: ProtocolEvent): void { + if (!this.options.onEvent) { + return; + } + const delivered = this.options.onEvent(event); + if (delivered !== false) { + this.stateStore.acknowledgeEvent(event); + } + } + + private assertContextPackageRef(reference: ContextPackageRef): void { + if (!reference.packageId.trim() || !Number.isInteger(reference.revision) || reference.revision < 0) { + throw new Error("PROTOCOL_CONTEXT_REF_INVALID"); + } + if (this.options.contextPackageExists && !this.options.contextPackageExists(reference)) { + throw new Error(`PROTOCOL_CONTEXT_REF_NOT_FOUND:${reference.packageId}@${reference.revision}`); + } + } +} + +const updateActionRecord = ( + actions: ProtocolRunState["actions"], + actionId: string, + replacement: ProtocolRunState["actions"][number] +): ProtocolRunState["actions"] => { + const index = actions.findIndex((action) => action.actionId === actionId); + if (index < 0) { + return [...actions, replacement]; + } + return actions.map((action, actionIndex) => actionIndex === index ? replacement : action); +}; + +const requireRequestedAction = ( + state: ProtocolRunState, + actionId: string, + actionName: string +): ProtocolRunState["actions"][number] => { + const action = state.actions.find((candidate) => candidate.actionId === actionId); + if (!action) { + throw new Error(`PROTOCOL_ACTION_NOT_STARTED:${actionId}:${actionName}`); + } + if (action.actionName !== actionName) { + throw new Error(`PROTOCOL_ACTION_NAME_MISMATCH:${actionId}:${action.actionName}:${actionName}`); + } + if (action.status !== "requested") { + throw new Error(`PROTOCOL_ACTION_ALREADY_SETTLED:${actionId}:${action.status}`); + } + return action; +}; + +const latestContextPackageRef = ( + current: ContextPackageRef, + candidate: ContextPackageRef +): ContextPackageRef => { + if (current.packageId !== candidate.packageId) { + throw new Error(`PROTOCOL_CONTEXT_PACKAGE_MISMATCH:${current.packageId}:${candidate.packageId}`); + } + return current.revision >= candidate.revision ? current : candidate; +}; + +const isProtocolRevisionConflict = (error: unknown): boolean => + error instanceof Error && error.message.startsWith("PROTOCOL_REVISION_CONFLICT:"); diff --git a/packages/agent-runtime/src/protocol/protocols/data-analysis.ts b/packages/agent-runtime/src/protocol/protocols/data-analysis.ts new file mode 100644 index 0000000..a2cbf59 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocols/data-analysis.ts @@ -0,0 +1,780 @@ +import { + createCoreAnalysisRequirements, + type AnalysisEvidenceBinding, + type AnalysisQueryAttempt, + type AnalysisReportedClaim, + type AnalysisRequirement, + type TaskRequirementLink +} from "../analysis-requirements.js"; +import { + resolveRequirementAssertions, + type AnalysisAssertion, + type AnalysisClaimValue, + type AnalysisScalar, + type AnalysisVerifiedValue +} from "../analysis-contract.js"; +import { validateSqlSemantics } from "../sql-semantic-validator.js"; +import type { AgentProtocolDefinition } from "../types.js"; + +const DATA_ACTIONS = new Set(["list_data_sources", "inspect_schema", "preview_table", "run_sql_readonly"]); + +export type DataAnalysisState = { + schemaInspected: boolean; + datasourceDialect?: string; + semanticResolved: boolean; + semanticMode?: string; + semanticTrust?: string; + semanticWarnings: string[]; + contractGrounded: boolean; + contractDatasourceRevision?: string; + contractSchemaId?: string; + contractGroundingFindings: Array<{ requirementId?: string; code?: string; message?: string }>; + queryPlanned: boolean; + queryValidated: boolean; + currentQueryValidated: boolean; + queryExecuted: boolean; + validationPassed: boolean; + currentEvidenceRefs: string[]; + evidenceRefs: string[]; + requirements: AnalysisRequirement[]; + queryAttempts: AnalysisQueryAttempt[]; + currentQueryAttemptId?: string; + evidenceBindings: AnalysisEvidenceBinding[]; + reportedClaims: AnalysisReportedClaim[]; + taskRequirementLinks: TaskRequirementLink[]; +}; + +export const createDataAnalysisProtocol = ( + availableActionNames: string[], + userRequirements: AnalysisRequirement[] = [] +): AgentProtocolDefinition => { + const commonActions = unique([ + ...availableActionNames.filter((actionName) => !DATA_ACTIONS.has(actionName)), + "protocol.handoff.propose" + ]); + return { + id: "data-analysis", + version: "1", + initialPhase: "scope", + phases: { + scope: { + allowedActions: unique([ + ...commonActions, + "list_data_sources", + "inspect_schema" + ]), + transitions: [{ targetPhase: "semantic_grounding", when: ({ state }) => state.schemaInspected }] + }, + semantic_grounding: { + allowedActions: unique([ + ...commonActions, + "inspect_schema", + "preview_table", + "semantic.context.resolve", + "analysis.contract.ground" + ]), + transitions: [{ + targetPhase: "query_planning", + when: ({ state }) => state.semanticResolved && state.contractGrounded + }] + }, + query_planning: { + allowedActions: unique([ + ...commonActions, + "inspect_schema", + "preview_table", + "semantic.context.resolve", + "data.query.plan", + "data.query.validate" + ]), + transitions: [{ targetPhase: "execution", when: ({ state }) => state.currentQueryValidated }] + }, + execution: { + allowedActions: unique([ + ...commonActions, + "inspect_schema", + "semantic.context.resolve", + "preview_table", + "run_sql_readonly", + "data.query.plan" + ]), + transitions: [ + { targetPhase: "query_planning", when: ({ actionName }) => actionName === "data.query.plan" }, + { + targetPhase: "validation", + when: ({ actionName, state }) => actionName === "run_sql_readonly" && state.queryExecuted + } + ] + }, + validation: { + allowedActions: unique([ + ...commonActions, + "inspect_schema", + "semantic.context.resolve", + "preview_table", + "data.query.plan", + "analysis.result.validate", + "analysis.evidence.bind", + "analysis.requirements.commit" + ]), + transitions: [ + { targetPhase: "query_planning", when: ({ actionName }) => actionName === "data.query.plan" }, + { + targetPhase: "synthesis", + when: ({ state }) => state.validationPassed && (state.currentEvidenceRefs ?? []).length > 0 + } + ] + }, + synthesis: { + allowedActions: unique([ + ...commonActions, + "inspect_schema", + "semantic.context.resolve", + "preview_table", + "data.query.plan", + "analysis.result.validate", + "analysis.evidence.bind", + "analysis.requirements.commit" + ]), + transitions: [{ + targetPhase: "query_planning", + when: ({ actionName }) => actionName === "data.query.plan" + }] + } + }, + createInitialState: () => ({ + schemaInspected: false, + semanticResolved: false, + semanticWarnings: [], + contractGrounded: false, + contractGroundingFindings: [], + queryPlanned: false, + queryValidated: false, + currentQueryValidated: false, + queryExecuted: false, + validationPassed: false, + currentEvidenceRefs: [], + evidenceRefs: [], + requirements: [...createCoreAnalysisRequirements(), ...cloneRequirements(userRequirements)], + queryAttempts: [], + evidenceBindings: [], + reportedClaims: [], + taskRequirementLinks: [] + }), + completionPolicy: ({ contextPackageRef, state }) => { + const requirementReasons = incompleteRequirementReasons(state); + if ( + state.semanticResolved + && state.contractGrounded + && state.queryPlanned + && state.currentQueryValidated + && state.queryExecuted + && state.validationPassed + && (state.currentEvidenceRefs ?? []).length > 0 + && requirementReasons.length === 0 + ) { + if (state.semanticMode === "fallback") { + return { + status: "degraded", + evaluatedContextPackageRef: contextPackageRef, + reasons: state.semanticWarnings.length > 0 + ? state.semanticWarnings + : ["LOCAL_SEMANTIC_FALLBACK"], + evidenceRefs: state.evidenceRefs + }; + } + return { + status: "completed", + evaluatedContextPackageRef: contextPackageRef, + evidenceRefs: state.evidenceRefs + }; + } + const missing = [ + ...(state.schemaInspected ? [] : ["SCHEMA_GROUNDING_REQUIRED"]), + ...(state.semanticResolved ? [] : ["SEMANTIC_GROUNDING_REQUIRED"]), + ...(state.contractGrounded ? [] : ["ANALYSIS_CONTRACT_GROUNDING_REQUIRED"]), + ...(state.queryPlanned ? [] : ["QUERY_PLAN_REQUIRED"]), + ...(state.currentQueryValidated ? [] : ["QUERY_VALIDATION_REQUIRED"]), + ...(state.queryExecuted ? [] : ["QUERY_EXECUTION_REQUIRED"]), + ...(state.validationPassed ? [] : ["RESULT_VALIDATION_REQUIRED"]), + ...((state.evidenceRefs ?? []).length > 0 ? [] : ["EVIDENCE_BINDING_REQUIRED"]), + ...requirementReasons + ]; + return { status: "continue", reasons: missing, allowedActions: allowedRecoveryActions(state) }; + } + }; +}; + +export const reduceDataAnalysisAction = ( + state: DataAnalysisState, + actionName: string, + result: unknown +): DataAnalysisState => { + if (actionName === "inspect_schema") { + const dialect = recordString(result, "dialect") ?? nestedString(result, "summary", "dialect"); + const schemaId = recordString(result, "schema_id") ?? recordString(result, "schemaId"); + return updateCoreRequirement({ + ...state, + schemaInspected: true, + contractGrounded: !hasUserRequirements(state) || state.contractGrounded, + ...(schemaId ? { contractSchemaId: schemaId } : {}), + ...(dialect ? { datasourceDialect: dialect } : {}) + }, "CORE_SCHEMA", "validated"); + } + if (actionName === "semantic.context.resolve") { + const semanticMode = recordString(result, "mode"); + const semanticTrust = recordString(result, "trust"); + const datasourceRevision = recordString(result, "datasourceRevision"); + const semanticContextChanged = state.contractGrounded && ( + (semanticMode !== undefined && state.semanticMode !== undefined && semanticMode !== state.semanticMode) + || (semanticTrust !== undefined && state.semanticTrust !== undefined && semanticTrust !== state.semanticTrust) + || (datasourceRevision !== undefined && state.contractDatasourceRevision !== undefined + && datasourceRevision !== state.contractDatasourceRevision) + ); + const next = { + ...state, + semanticResolved: semanticMode !== undefined && semanticMode !== "unavailable", + contractGrounded: !hasUserRequirements(state) || (state.contractGrounded && !semanticContextChanged), + ...(semanticMode ? { semanticMode } : {}), + ...(semanticTrust ? { semanticTrust } : {}), + semanticWarnings: recordStrings(result, "warnings") + }; + return next.semanticResolved ? updateCoreRequirement(next, "CORE_SEMANTIC", "validated") : next; + } + if (actionName === "analysis.contract.ground") { + const requirements = recordArray(result, "requirements").filter(isAnalysisRequirement); + const datasourceRevision = recordString(result, "datasourceRevision"); + const schemaId = recordString(result, "schema_id") ?? recordString(result, "schemaId"); + return { + ...state, + contractGrounded: requirements.length > 0, + ...(datasourceRevision ? { contractDatasourceRevision: datasourceRevision } : {}), + ...(schemaId ? { contractSchemaId: schemaId } : {}), + contractGroundingFindings: recordArray(result, "findings").map((finding) => ({ + ...(recordString(finding, "requirementId") + ? { requirementId: recordString(finding, "requirementId") as string } + : {}), + ...(recordString(finding, "code") ? { code: recordString(finding, "code") as string } : {}), + ...(recordString(finding, "message") ? { message: recordString(finding, "message") as string } : {}) + })), + ...(requirements.length > 0 ? { requirements: cloneRequirements(requirements) } : {}) + }; + } + if (actionName === "data.query.plan") { + const assertionIds = recordStrings(result, "assertion_ids"); + const explicitRequirementIds = recordStrings(result, "requirement_ids"); + const requirementIds = explicitRequirementIds.length > 0 + ? explicitRequirementIds + : deriveRequirementIdsFromAssertions(state, assertionIds); + assertRequirementIds(state, requirementIds); + const userRequirements = normalizedRequirements(state).filter((requirement) => requirement.source === "user"); + if (userRequirements.length > 0 && requirementIds.length === 0) { + throw new Error("ANALYSIS_REQUIREMENT_IDS_REQUIRED"); + } + const queryAttempts = normalizedQueryAttempts(state); + const attemptId = `Q${queryAttempts.length + 1}`; + const sql = recordString(result, "sql"); + const selectedRequirements = normalizedRequirements(state).filter((requirement) => + requirementIds.includes(requirement.id)); + const structuredAssertions = selectedRequirements.flatMap((requirement) => requirement.assertions ?? []) + .filter((assertion) => assertion.kind !== "manual"); + if (structuredAssertions.length > 0 && assertionIds.length === 0) { + throw new Error(`ANALYSIS_ASSERTION_IDS_REQUIRED:${requirementIds[0] ?? "unknown"}`); + } + const assertions = assertionIds.length > 0 + ? resolveRequirementAssertions(normalizedRequirements(state), requirementIds, assertionIds) + : selectedRequirements.flatMap((requirement) => requirement.assertions ?? []) + .filter((assertion) => assertion.kind === "manual"); + const attempt: AnalysisQueryAttempt = { + id: attemptId, + requirementIds, + assertionIds, + assertions, + ...(sql ? { sql } : {}), + expectedColumns: recordStrings(result, "expected_columns"), + status: "planned", + valid: false, + resultFields: [], + validationFindings: [], + resultValidationFindings: [], + verifiedValues: [] + }; + return updateRequirements({ + ...state, + queryPlanned: true, + currentQueryValidated: false, + queryExecuted: false, + validationPassed: false, + currentEvidenceRefs: [], + currentQueryAttemptId: attemptId, + queryAttempts: [...queryAttempts, attempt] + }, requirementIds, (requirement) => ({ + ...requirement, + status: advanceStatus(requirement.status, "queried"), + queryAttemptIds: unique([...requirement.queryAttemptIds, attemptId]) + })); + } + if (actionName === "data.query.validate") { + const attempt = currentAttempt(state); + const runtimeValid = recordBoolean(result, "valid") === true; + const semanticFindings = runtimeValid && attempt?.sql + && (attempt.assertions ?? []).some((assertion) => assertion.kind !== "manual") + ? validateSqlSemantics(attempt.sql, state.datasourceDialect, attempt.assertions) + : []; + const runtimeFindings = recordStrings(result, "reasons").map((reason) => ({ + code: reason, + message: reason, + severity: "error" as const + })); + const validationFindings = [...runtimeFindings, ...semanticFindings]; + const valid = runtimeValid && !validationFindings.some((finding) => finding.severity === "error"); + let next = { + ...state, + queryValidated: state.queryValidated || valid, + currentQueryValidated: valid, + queryAttempts: updateCurrentAttempt(state, (attempt) => ({ + ...attempt, + status: valid ? "validated" : "planned", + valid, + validationFindings + })) + }; + if (valid) { + next = updateCoreRequirement(next, "CORE_QUERY", "validated"); + } + return next; + } + if (actionName === "run_sql_readonly") { + const artifactId = nestedArtifactId(result); + const auditLogId = nestedString(result, "result", "audit_log_id"); + const resultFields = nestedStrings(result, "result", "columns"); + return { + ...state, + queryExecuted: true, + validationPassed: false, + currentEvidenceRefs: artifactId ? [artifactId] : [], + evidenceRefs: artifactId ? unique([...(state.evidenceRefs ?? []), artifactId]) : state.evidenceRefs, + queryAttempts: updateCurrentAttempt(state, (attempt) => ({ + ...attempt, + status: "executed", + ...(artifactId ? { artifactId } : {}), + ...(auditLogId ? { auditLogId } : {}), + resultFields + })) + }; + } + if (actionName === "analysis.result.validate") { + const valid = recordBoolean(result, "valid") === true; + const resultValidationFindings = recordArray(result, "validation_findings") + .filter(isAnalysisValidationFinding); + const verifiedValues = recordArray(result, "verified_values").filter(isAnalysisVerifiedValue); + let next = { + ...state, + validationPassed: valid, + queryAttempts: updateCurrentAttempt(state, (attempt) => ({ + ...attempt, + resultValidationFindings, + verifiedValues + })) + }; + if (valid) { + next = updateCoreRequirement(next, "CORE_RESULT", "validated"); + const requirementIds = currentAttempt(state)?.requirementIds ?? []; + next = updateRequirements(next, requirementIds, (requirement) => ({ + ...requirement, + status: advanceStatus(requirement.status, "validated") + })); + } + return next; + } + if (actionName === "analysis.evidence.bind") { + const evidenceRefs = recordStrings(result, "evidence_refs"); + const attempt = currentAttempt(state); + const artifactId = recordString(result, "artifact_id") ?? attempt?.artifactId ?? evidenceRefs[0]; + const auditLogId = recordString(result, "audit_log_id") ?? attempt?.auditLogId; + const resultFields = recordStrings(result, "result_fields"); + if ((attempt?.requirementIds.length ?? 0) > 0 && (!artifactId || !auditLogId || !state.validationPassed)) { + throw new Error("ANALYSIS_EVIDENCE_BINDING_INCOMPLETE"); + } + const bindings = createEvidenceBindings(state, attempt, artifactId, auditLogId, resultFields); + let next = { + ...state, + currentEvidenceRefs: unique([...(state.currentEvidenceRefs ?? []), ...evidenceRefs]), + evidenceRefs: unique([...(state.evidenceRefs ?? []), ...evidenceRefs]), + evidenceBindings: [...normalizedEvidenceBindings(state), ...bindings], + queryAttempts: updateCurrentAttempt(state, (queryAttempt) => ({ ...queryAttempt, status: "evidenced" })) + }; + next = updateRequirements(next, attempt?.requirementIds ?? [], (requirement) => ({ + ...requirement, + status: advanceStatus(requirement.status, "evidenced"), + evidenceBindingIds: unique([ + ...requirement.evidenceBindingIds, + ...bindings.filter((binding) => binding.requirementId === requirement.id).map((binding) => binding.id) + ]) + })); + return updateCoreRequirement(next, "CORE_EVIDENCE", "evidenced"); + } + if (actionName === "analysis.requirements.commit") { + return commitReportedClaims(state, result); + } + if (actionName === "task_write" || actionName === "task_update" || actionName === "task_complete") { + return linkTasksToRequirements(state, result); + } + return state; +}; + +const allowedRecoveryActions = (state: DataAnalysisState): string[] => { + if (!state.schemaInspected) return ["inspect_schema", "semantic.context.resolve"]; + if (!state.semanticResolved) return ["semantic.context.resolve"]; + if (!state.contractGrounded) return ["analysis.contract.ground"]; + if (!state.queryPlanned) return ["data.query.plan"]; + if (!state.currentQueryValidated) return ["data.query.validate"]; + if (!state.queryExecuted) { + return ["inspect_schema", "preview_table", "data.query.plan", "data.query.validate", "run_sql_readonly"]; + } + if (!state.validationPassed) return ["analysis.result.validate", "run_sql_readonly"]; + if ((state.currentEvidenceRefs ?? []).length === 0) return ["analysis.evidence.bind"]; + if (incompleteRequirementReasons(state).length > 0) return ["data.query.plan", "analysis.requirements.commit"]; + return []; +}; + +const hasUserRequirements = (state: DataAnalysisState): boolean => + normalizedRequirements(state).some((requirement) => requirement.source === "user"); + +const deriveRequirementIdsFromAssertions = (state: DataAnalysisState, assertionIds: string[]): string[] => unique( + normalizedRequirements(state) + .filter((requirement) => requirement.assertions.some((assertion) => assertionIds.includes(assertion.id))) + .map((requirement) => requirement.id) +); + +const isAnalysisRequirement = (value: unknown): value is AnalysisRequirement => + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof (value as Record).id === "string" + && Array.isArray((value as Record).assertions); + +const incompleteRequirementReasons = (state: DataAnalysisState): string[] => normalizedRequirements(state).flatMap( + (requirement) => { + if (!requirement.required) return []; + if (requirement.source === "protocol") { + return requirement.status === "pending" || requirement.status === "queried" + ? [`ANALYSIS_CORE_REQUIREMENT_PENDING:${requirement.id}`] + : []; + } + if (requirement.status === "reported") return []; + return [requirement.status === "evidenced" + ? `ANALYSIS_REQUIREMENT_NOT_REPORTED:${requirement.id}` + : `ANALYSIS_REQUIREMENT_PENDING:${requirement.id}`]; + } +); + +const updateCoreRequirement = ( + state: DataAnalysisState, + requirementId: string, + status: AnalysisRequirement["status"] +): DataAnalysisState => updateRequirements(state, [requirementId], (requirement) => ({ + ...requirement, + status: advanceStatus(requirement.status, status) +})); + +const updateRequirements = ( + state: DataAnalysisState, + requirementIds: string[], + update: (requirement: AnalysisRequirement) => AnalysisRequirement +): DataAnalysisState => { + const idSet = new Set(requirementIds); + return { + ...state, + requirements: normalizedRequirements(state).map((requirement) => idSet.has(requirement.id) + ? update(requirement) + : requirement) + }; +}; + +const assertRequirementIds = (state: DataAnalysisState, requirementIds: string[]): void => { + const knownIds = new Set(normalizedRequirements(state).map((requirement) => requirement.id)); + for (const requirementId of requirementIds) { + if (!knownIds.has(requirementId) || requirementId.startsWith("CORE_")) { + throw new Error(`ANALYSIS_REQUIREMENT_NOT_FOUND:${requirementId}`); + } + } +}; + +const updateCurrentAttempt = ( + state: DataAnalysisState, + update: (attempt: AnalysisQueryAttempt) => AnalysisQueryAttempt +): AnalysisQueryAttempt[] => normalizedQueryAttempts(state).map((attempt) => + attempt.id === state.currentQueryAttemptId ? update(attempt) : attempt); + +const currentAttempt = (state: DataAnalysisState): AnalysisQueryAttempt | undefined => + normalizedQueryAttempts(state).find((attempt) => attempt.id === state.currentQueryAttemptId); + +const createEvidenceBindings = ( + state: DataAnalysisState, + attempt: AnalysisQueryAttempt | undefined, + artifactId: string | undefined, + auditLogId: string | undefined, + resultFields: string[] +): AnalysisEvidenceBinding[] => { + if (!attempt || !artifactId || !auditLogId || !state.validationPassed) return []; + const offset = normalizedEvidenceBindings(state).length; + return attempt.requirementIds.map((requirementId, index) => ({ + id: `E${offset + index + 1}`, + requirementId, + queryAttemptId: attempt.id, + artifactId, + auditLogId, + resultFields: resultFields.length > 0 ? resultFields : attempt.resultFields, + validationStatus: "passed" + })); +}; + +const commitReportedClaims = (state: DataAnalysisState, result: unknown): DataAnalysisState => { + const claims = recordArray(result, "claims"); + let next = { ...state, reportedClaims: [...normalizedReportedClaims(state)] }; + for (const value of claims) { + const requirementId = recordString(value, "requirement_id"); + const claim = recordString(value, "claim"); + const evidenceBindingIds = recordStrings(value, "evidence_binding_ids"); + const evidenceRefs = recordStrings(value, "evidence_refs"); + const evidenceRequirementIds = recordStrings(value, "evidence_requirement_ids"); + if (!requirementId || !claim) throw new Error("ANALYSIS_REQUIREMENT_CLAIM_INVALID"); + assertRequirementIds(next, [requirementId]); + assertRequirementIds(next, evidenceRequirementIds); + const requirement = normalizedRequirements(next).find((candidate) => candidate.id === requirementId); + const requirementBindings = normalizedEvidenceBindings(next).filter((binding) => + binding.requirementId === requirementId); + const sourceBindings = normalizedEvidenceBindings(next).filter((binding) => + evidenceRequirementIds.includes(binding.requirementId)); + let candidateBindings = uniqueBindings([...requirementBindings, ...sourceBindings]); + if (candidateBindings.length === 0 && (requirement?.kind === "validation" || requirement?.kind === "decision")) { + candidateBindings = normalizedEvidenceBindings(next); + } + const hasExplicitEvidence = evidenceBindingIds.length > 0 || evidenceRefs.length > 0; + let validBindings = hasExplicitEvidence + ? candidateBindings.filter((binding) => + evidenceBindingIds.includes(binding.id) || evidenceRefs.includes(binding.artifactId)) + : candidateBindings; + if (validBindings.length === 0 && candidateBindings.length > 0) { + validBindings = candidateBindings; + } + if (validBindings.length === 0) { + const invalidId = evidenceBindingIds[0]; + const invalidRef = evidenceRefs[0]; + throw new Error( + `ANALYSIS_REQUIREMENT_EVIDENCE_INVALID:${requirementId}:${invalidId ?? invalidRef ?? "missing"}` + ); + } + const values = recordArray(value, "values").map(parseClaimValue); + const boundAttemptIds = new Set(validBindings.map((binding) => binding.queryAttemptId)); + const boundAttempts = normalizedQueryAttempts(next).filter((attempt) => boundAttemptIds.has(attempt.id)); + const requirementAssertionIds = new Set(boundAttempts.flatMap((attempt) => attempt.assertions + .filter((assertion) => assertion.requirementId === requirementId) + .map((assertion) => assertion.id))); + const verifiedValues = boundAttempts.flatMap((attempt) => attempt.verifiedValues ?? []) + .filter((verifiedValue) => requirementAssertionIds.has(verifiedValue.assertionId)); + const requiredValueSpecs = (requirement?.assertions ?? []).flatMap((assertion) => + assertion.required && assertion.kind !== "manual" + ? assertion.claimValues.filter((spec) => spec.required).map((spec) => ({ + assertionId: assertion.id, + name: spec.name + })) + : []); + for (const spec of requiredValueSpecs) { + const verified = verifiedValues.some((value) => + value.assertionId === spec.assertionId && value.name === spec.name); + if (!verified) { + throw new Error( + `ANALYSIS_CLAIM_VALUE_NOT_VERIFIED:${requirementId}:${spec.name}: execute and bind evidence for assertion ` + + `${spec.assertionId} before committing.` + ); + } + } + const requiredValueNames = unique(requiredValueSpecs.map((spec) => spec.name)); + validateClaimValues(requirementId, values, verifiedValues, requiredValueNames); + const claimId = `C${next.reportedClaims.length + 1}`; + next.reportedClaims.push({ + id: claimId, + requirementId, + claim, + evidenceBindingIds: validBindings.map((binding) => binding.id), + values + }); + next = updateRequirements(next, [requirementId], (requirement) => ({ + ...requirement, + status: "reported", + reportedClaimIds: unique([...requirement.reportedClaimIds, claimId]) + })); + } + return next; +}; + +const uniqueBindings = (bindings: AnalysisEvidenceBinding[]): AnalysisEvidenceBinding[] => [...new Map( + bindings.map((binding) => [binding.id, binding]) +).values()]; + +const linkTasksToRequirements = (state: DataAnalysisState, result: unknown): DataAnalysisState => { + const tasks = recordArray(result, "tasks"); + let requirements = normalizedRequirements(state); + const links = [...normalizedTaskLinks(state)]; + for (const task of tasks) { + const taskId = recordString(task, "id"); + const content = recordString(task, "content"); + if (!taskId || !content) continue; + const requirementIds = requirements + .filter((requirement) => requirement.source === "user" && content.includes(requirement.id)) + .map((requirement) => requirement.id); + if (requirementIds.length === 0) continue; + const existing = links.find((link) => link.taskId === taskId); + if (existing) { + existing.requirementIds = unique([...existing.requirementIds, ...requirementIds]); + } else { + links.push({ taskId, requirementIds }); + } + requirements = requirements.map((requirement) => requirementIds.includes(requirement.id) + ? { ...requirement, taskIds: unique([...requirement.taskIds, taskId]) } + : requirement); + } + return { ...state, requirements, taskRequirementLinks: links }; +}; + +const advanceStatus = ( + current: AnalysisRequirement["status"], + target: AnalysisRequirement["status"] +): AnalysisRequirement["status"] => { + const order: AnalysisRequirement["status"][] = ["pending", "queried", "validated", "evidenced", "reported"]; + return order.indexOf(target) > order.indexOf(current) ? target : current; +}; + +const cloneRequirements = (requirements: AnalysisRequirement[]): AnalysisRequirement[] => requirements.map( + (requirement) => ({ + ...requirement, + acceptanceCriteria: [...requirement.acceptanceCriteria], + assertions: cloneAssertions(requirement.assertions ?? []), + taskIds: [...requirement.taskIds], + queryAttemptIds: [...requirement.queryAttemptIds], + evidenceBindingIds: [...requirement.evidenceBindingIds], + reportedClaimIds: [...requirement.reportedClaimIds] + }) +); + +const cloneAssertions = (assertions: AnalysisAssertion[]): AnalysisAssertion[] => assertions.map((assertion) => ({ + ...assertion, + sourceTables: [...assertion.sourceTables], + dimensions: [...assertion.dimensions], + sqlConstraints: structuredClone(assertion.sqlConstraints), + resultChecks: structuredClone(assertion.resultChecks), + claimValues: structuredClone(assertion.claimValues) +})); + +const normalizedRequirements = (state: DataAnalysisState): AnalysisRequirement[] => state.requirements ?? []; +const normalizedQueryAttempts = (state: DataAnalysisState): AnalysisQueryAttempt[] => state.queryAttempts ?? []; +const normalizedEvidenceBindings = (state: DataAnalysisState): AnalysisEvidenceBinding[] => state.evidenceBindings ?? []; +const normalizedReportedClaims = (state: DataAnalysisState): AnalysisReportedClaim[] => state.reportedClaims ?? []; +const normalizedTaskLinks = (state: DataAnalysisState): TaskRequirementLink[] => state.taskRequirementLinks ?? []; + +const nestedArtifactId = (value: unknown): string | undefined => + recordString(value, "artifact_id") ?? recordString(recordValue(value, "result"), "artifact_id"); + +const nestedString = (value: unknown, parent: string, key: string): string | undefined => + recordString(recordValue(value, parent), key); + +const nestedStrings = (value: unknown, parent: string, key: string): string[] => + recordStrings(recordValue(value, parent), key); + +const recordValue = (value: unknown, key: string): unknown => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record)[key] + : undefined; + +const recordString = (value: unknown, key: string): string | undefined => { + const field = recordValue(value, key); + return typeof field === "string" && field.length > 0 ? field : undefined; +}; + +const recordBoolean = (value: unknown, key: string): boolean | undefined => { + const field = recordValue(value, key); + return typeof field === "boolean" ? field : undefined; +}; + +const recordStrings = (value: unknown, key: string): string[] => { + const field = recordValue(value, key); + return Array.isArray(field) ? field.filter((item): item is string => typeof item === "string") : []; +}; + +const recordArray = (value: unknown, key: string): unknown[] => { + const field = recordValue(value, key); + return Array.isArray(field) ? field : []; +}; + +const parseClaimValue = (value: unknown): AnalysisClaimValue => { + const name = recordString(value, "name"); + const scalar = recordValue(value, "value"); + const unit = recordString(value, "unit"); + if (!name || !isAnalysisScalar(scalar)) throw new Error("ANALYSIS_CLAIM_VALUE_INVALID"); + return { name, value: scalar, ...(unit ? { unit } : {}) }; +}; + +const validateClaimValues = ( + requirementId: string, + values: AnalysisClaimValue[], + verifiedValues: AnalysisVerifiedValue[], + requiredValueNames: string[] +): void => { + const submittedNames = new Set(); + for (const value of values) { + if (submittedNames.has(value.name)) throw new Error(`ANALYSIS_CLAIM_VALUE_DUPLICATE:${value.name}`); + submittedNames.add(value.name); + const verified = verifiedValues.find((candidate) => candidate.name === value.name); + if (!verified) { + const allowedNames = unique(verifiedValues.map((candidate) => candidate.name)).join(", ") || "none"; + throw new Error( + `ANALYSIS_CLAIM_VALUE_UNKNOWN:${requirementId}:${value.name}: use an exact verified name; ` + + `allowed names: ${allowedNames}.` + ); + } + const unitMatches = verified.unit === value.unit; + if (!unitMatches || !claimScalarsEqual(value.value, verified.value, verified.tolerance)) { + throw new Error( + `ANALYSIS_CLAIM_VALUE_MISMATCH:${requirementId}:${value.name}: ` + + `expected ${formatVerifiedClaimValue(verified)}; received ${formatClaimValue(value)}.` + ); + } + } + for (const name of requiredValueNames) { + if (submittedNames.has(name)) continue; + const verified = verifiedValues.find((candidate) => candidate.name === name); + const expected = verified ? formatVerifiedClaimValue(verified) : "the verified value emitted by result validation"; + throw new Error(`ANALYSIS_CLAIM_VALUE_REQUIRED:${requirementId}:${name}: submit ${expected}.`); + } +}; + +const formatVerifiedClaimValue = (value: AnalysisVerifiedValue): string => + `${formatClaimValue(value)}, tolerance=${value.tolerance}`; + +const formatClaimValue = (value: AnalysisClaimValue | AnalysisVerifiedValue): string => + `value=${JSON.stringify(value.value)}, ${value.unit === undefined ? "no unit" : `unit=${JSON.stringify(value.unit)}`}`; + +const claimScalarsEqual = (left: AnalysisScalar, right: AnalysisScalar, tolerance: number): boolean => + typeof left === "number" && typeof right === "number" + ? Math.abs(left - right) <= tolerance + : left === right; + +const isAnalysisScalar = (value: unknown): value is AnalysisScalar => + value === null || ["string", "number", "boolean"].includes(typeof value); + +const isAnalysisValidationFinding = (value: unknown): value is AnalysisQueryAttempt["validationFindings"][number] => + typeof value === "object" && value !== null && !Array.isArray(value) + && typeof (value as Record).code === "string" + && typeof (value as Record).message === "string" + && ["error", "warning"].includes(String((value as Record).severity)); + +const isAnalysisVerifiedValue = (value: unknown): value is AnalysisQueryAttempt["verifiedValues"][number] => + typeof value === "object" && value !== null && !Array.isArray(value) + && typeof (value as Record).name === "string" + && typeof (value as Record).assertionId === "string" + && typeof (value as Record).tolerance === "number" + && Object.hasOwn(value, "value"); + +const unique = (values: T[]): T[] => [...new Set(values)]; diff --git a/packages/agent-runtime/src/protocol/protocols/formal-protocols.test.ts b/packages/agent-runtime/src/protocol/protocols/formal-protocols.test.ts new file mode 100644 index 0000000..4bcbda8 --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocols/formal-protocols.test.ts @@ -0,0 +1,488 @@ +import { describe, expect, it } from "vitest"; + +import { createUserAnalysisRequirements } from "../analysis-requirements.js"; +import { createDataAnalysisProtocol, reduceDataAnalysisAction } from "./data-analysis.js"; +import { createGeneralTaskProtocol, reduceGeneralTaskAction } from "./general-task.js"; + +const contextPackageRef = { packageId: "context-1", revision: 0 }; + +describe("formal protocols", () => { + it("keeps data tools out of the general-task protocol", () => { + const protocol = createGeneralTaskProtocol([ + "retrieve_knowledge", + "read_file", + "inspect_schema", + "run_sql_readonly" + ]); + + expect(protocol.initialPhase).toBe("understand"); + expect(protocol.phases.understand?.allowedActions).toContain("retrieve_knowledge"); + expect(protocol.phases.understand?.allowedActions).not.toContain("inspect_schema"); + expect(protocol.phases.gather?.allowedActions).not.toContain("run_sql_readonly"); + expect(protocol.phases.answer?.allowedActions).toEqual([]); + }); + + it("completes a general task only after its answer is committed", () => { + const protocol = createGeneralTaskProtocol([]); + const initial = protocol.createInitialState({ contextPackageRef, runId: "run-1" }); + expect(protocol.completionPolicy({ contextPackageRef, state: initial }).status).toBe("continue"); + + const committed = reduceGeneralTaskAction(initial, "general.answer.commit", { messageId: "message-1" }); + expect(protocol.completionPolicy({ contextPackageRef, state: committed }).status).toBe("completed"); + }); + + it("rejects a general answer commit without a message id", () => { + expect(() => reduceGeneralTaskAction({}, "general.answer.commit", {})) + .toThrow("GENERAL_ANSWER_MESSAGE_MISSING"); + }); + + it("does not allow SQL before schema grounding", () => { + const protocol = createDataAnalysisProtocol(["inspect_schema", "run_sql_readonly"]); + + expect(protocol.initialPhase).toBe("scope"); + expect(protocol.phases.scope?.allowedActions).toContain("inspect_schema"); + expect(protocol.phases.semantic_grounding?.allowedActions).not.toContain("run_sql_readonly"); + expect(protocol.phases.query_planning?.allowedActions).toContain("data.query.validate"); + expect(protocol.phases.query_planning?.allowedActions).not.toContain("run_sql_readonly"); + expect(protocol.phases.execution?.allowedActions).toContain("run_sql_readonly"); + }); + + it("requires validated evidence before completing data analysis", () => { + const protocol = createDataAnalysisProtocol(["inspect_schema", "run_sql_readonly"]); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-1" }); + state = reduceDataAnalysisAction(state, "inspect_schema", {}); + state = reduceDataAnalysisAction(state, "semantic.context.resolve", { + mode: "live", + trust: "verified", + warnings: [] + }); + state = reduceDataAnalysisAction(state, "data.query.plan", { sql: "select 1" }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "run_sql_readonly", { artifact_id: "artifact-1" }); + expect(protocol.completionPolicy({ contextPackageRef, state }).status).toBe("continue"); + + state = reduceDataAnalysisAction(state, "analysis.result.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "analysis.evidence.bind", { evidence_refs: ["artifact-1"] }); + expect(protocol.completionPolicy({ contextPackageRef, state }).status).toBe("completed"); + }); + + it("discloses local semantic fallback as degraded completion", () => { + const protocol = createDataAnalysisProtocol([]); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-1" }); + state = reduceDataAnalysisAction(state, "inspect_schema", {}); + state = reduceDataAnalysisAction(state, "semantic.context.resolve", { + mode: "fallback", + trust: "verified", + warnings: ["LOCAL_SEMANTIC_LIMITED_TO_PHYSICAL_SCHEMA"] + }); + state = reduceDataAnalysisAction(state, "data.query.plan", {}); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "run_sql_readonly", { artifact_id: "artifact-1" }); + state = reduceDataAnalysisAction(state, "analysis.result.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "analysis.evidence.bind", { evidence_refs: ["artifact-1"] }); + + expect(protocol.completionPolicy({ contextPackageRef, state })).toMatchObject({ + status: "degraded", + reasons: ["LOCAL_SEMANTIC_LIMITED_TO_PHYSICAL_SCHEMA"] + }); + }); + + it("preserves a grounded contract when the same schema is inspected again", () => { + const requirements = createUserAnalysisRequirements([{ + kind: "metric", + description: "统计订单数", + acceptanceCriteria: ["返回订单总数"], + assertions: [{ + kind: "metric", + description: "订单数", + sourceTables: ["orders"], + claimValues: [{ name: "order_count", field: "order_count", required: true }] + }] + }]); + const protocol = createDataAnalysisProtocol([], requirements); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-reinspect" }); + state = reduceDataAnalysisAction(state, "inspect_schema", { schema_id: "schema-orders" }); + state = reduceDataAnalysisAction(state, "semantic.context.resolve", { mode: "live", trust: "verified" }); + state = reduceDataAnalysisAction(state, "analysis.contract.ground", { + schema_id: "schema-orders", + datasourceRevision: "revision-1", + requirements + }); + + const refreshedSnapshot = reduceDataAnalysisAction(state, "inspect_schema", { schema_id: "schema-orders-refresh" }); + const sameRevision = reduceDataAnalysisAction(refreshedSnapshot, "semantic.context.resolve", { + mode: "live", + trust: "verified", + datasourceRevision: "revision-1" + }); + const changedRevision = reduceDataAnalysisAction(refreshedSnapshot, "semantic.context.resolve", { + mode: "live", + trust: "verified", + datasourceRevision: "revision-2" + }); + + expect(refreshedSnapshot.contractGrounded).toBe(true); + expect(sameRevision.contractGrounded).toBe(true); + expect(changedRevision.contractGrounded).toBe(false); + }); + + it("keeps a later query attempt pending until it has its own validated evidence", () => { + const protocol = createDataAnalysisProtocol([]); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-1" }); + state = reduceDataAnalysisAction(state, "inspect_schema", {}); + state = reduceDataAnalysisAction(state, "semantic.context.resolve", { mode: "live", warnings: [] }); + state = reduceDataAnalysisAction(state, "data.query.plan", { sql: "select 1" }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "run_sql_readonly", { artifact_id: "artifact-1" }); + state = reduceDataAnalysisAction(state, "analysis.result.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "analysis.evidence.bind", { evidence_refs: ["artifact-1"] }); + state = reduceDataAnalysisAction(state, "data.query.plan", { sql: "delete from orders" }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: false }); + + expect(state).toMatchObject({ + currentQueryValidated: false, + queryExecuted: false, + validationPassed: false + }); + expect(protocol.completionPolicy({ contextPackageRef, state }).status).toBe("continue"); + }); + + it("keeps SQL semantic failures on the original attempt and validates a repaired attempt", () => { + const requirements = createUserAnalysisRequirements([{ + kind: "metric", + description: "统计完整验证期订单数", + acceptanceCriteria: ["包含12月31日"], + assertions: [{ + kind: "metric", + description: "验证期订单数", + sourceTables: ["orders"], + sqlConstraints: [{ + kind: "time_range", + column: "order_date", + start: "2023-07-01", + end: "2023-12-31", + endInclusive: true + }] + }] + }]); + const protocol = createDataAnalysisProtocol([], requirements); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-semantic-repair" }); + state = reduceDataAnalysisAction(state, "inspect_schema", { dialect: "sqlite" }); + state = reduceDataAnalysisAction(state, "semantic.context.resolve", { mode: "fallback", warnings: [] }); + state = reduceDataAnalysisAction(state, "data.query.plan", { + requirement_ids: ["R1"], + assertion_ids: ["R1.A1"], + sql: "SELECT COUNT(*) AS orders FROM orders WHERE order_date >= '2023-07-01' AND order_date < '2023-12-31'" + }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true, reasons: [] }); + + expect(state.currentQueryValidated).toBe(false); + expect(state.queryAttempts[0]).toMatchObject({ + id: "Q1", + valid: false, + validationFindings: [{ code: "SQL_SEMANTIC_TIME_END_MISSING:order_date" }] + }); + + state = reduceDataAnalysisAction(state, "data.query.plan", { + requirement_ids: ["R1"], + assertion_ids: ["R1.A1"], + sql: "SELECT COUNT(*) AS orders FROM orders WHERE order_date >= '2023-07-01' AND order_date < '2024-01-01'" + }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true, reasons: [] }); + + expect(state.currentQueryValidated).toBe(true); + expect(state.queryAttempts).toMatchObject([ + { id: "Q1", valid: false }, + { id: "Q2", valid: true, validationFindings: [] } + ]); + }); + + it("requires every user requirement to bind validated evidence and a reported claim", () => { + const requirements = createUserAnalysisRequirements([ + { kind: "data_quality", description: "核对利润公式", acceptanceCriteria: ["报告错误数"] }, + { kind: "metric", description: "计算新增利润", acceptanceCriteria: ["精确到分"] } + ]); + const protocol = createDataAnalysisProtocol([], requirements); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-requirements" }); + state = reduceDataAnalysisAction(state, "inspect_schema", {}); + state = reduceDataAnalysisAction(state, "semantic.context.resolve", { mode: "live", warnings: [] }); + state = reduceDataAnalysisAction(state, "data.query.plan", { + requirement_ids: ["R1"], + expected_columns: ["formula_mismatches"], + sql: "select 0 as formula_mismatches" + }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "run_sql_readonly", { + result: { + artifact_id: "artifact-1", + audit_log_id: "audit-1", + columns: ["formula_mismatches"], + rows: [[0]], + row_count: 1 + } + }); + state = reduceDataAnalysisAction(state, "analysis.result.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "analysis.evidence.bind", { + artifact_id: "artifact-1", + audit_log_id: "audit-1", + evidence_refs: ["artifact-1"], + result_fields: ["formula_mismatches"] + }); + + expect(state.requirements.find((requirement) => requirement.id === "R1")?.status).toBe("evidenced"); + expect(protocol.completionPolicy({ contextPackageRef, state })).toMatchObject({ + status: "continue", + reasons: expect.arrayContaining(["ANALYSIS_REQUIREMENT_NOT_REPORTED:R1", "ANALYSIS_REQUIREMENT_PENDING:R2"]) + }); + + const evidenceId = state.evidenceBindings.find((binding) => binding.requirementId === "R1")?.id; + expect(evidenceId).toBeDefined(); + const stateWithExtraEvidence = reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R1", + claim: "利润公式错误数为 0", + evidence_refs: ["artifact-1", "unrelated-artifact"] + }] + }); + expect(stateWithExtraEvidence.requirements.find((requirement) => requirement.id === "R1")?.status) + .toBe("reported"); + const stateWithResolvedEvidence = reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ requirement_id: "R1", claim: "利润公式错误数为 0" }] + }); + expect(stateWithResolvedEvidence.reportedClaims[0]?.evidenceBindingIds).toEqual([evidenceId]); + const stateWithSourceRequirement = reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R2", + claim: "新增利润基于已验证的利润公式", + evidence_requirement_ids: ["R1"] + }] + }); + expect(stateWithSourceRequirement.reportedClaims[0]?.evidenceBindingIds).toEqual([evidenceId]); + state = reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ requirement_id: "R1", claim: "利润公式错误数为 0", evidence_binding_ids: [evidenceId] }] + }); + expect(state.requirements.find((requirement) => requirement.id === "R1")?.status).toBe("reported"); + expect(protocol.completionPolicy({ contextPackageRef, state }).status).toBe("continue"); + }); + + it("links existing Mastra tasks to requirements without treating task completion as evidence", () => { + const requirements = createUserAnalysisRequirements([ + { kind: "comparison", description: "比较冻结组合", acceptanceCriteria: [] } + ]); + const protocol = createDataAnalysisProtocol([], requirements); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-task-link" }); + + state = reduceDataAnalysisAction(state, "task_write", { + tasks: [{ id: "task-holdout", content: "[R1] 比较冻结组合", activeForm: "比较中", status: "pending" }] + }); + state = reduceDataAnalysisAction(state, "task_complete", { + tasks: [{ id: "task-holdout", content: "[R1] 比较冻结组合", activeForm: "比较中", status: "completed" }] + }); + + expect(state.taskRequirementLinks).toEqual([{ taskId: "task-holdout", requirementIds: ["R1"] }]); + expect(state.requirements.find((requirement) => requirement.id === "R1")).toMatchObject({ + status: "pending", + taskIds: ["task-holdout"] + }); + }); + + it("accepts claim values only when they match deterministic result verification", () => { + const requirements = createUserAnalysisRequirements([{ + kind: "metric", + description: "计算订单转化率", + acceptanceCriteria: ["结果可由分子分母复算"], + assertions: [{ + kind: "metric", + description: "订单转化率", + resultChecks: [{ + kind: "ratio", + required: true, + value: { field: "conversion_rate" }, + numerator: { field: "converted" }, + denominator: { field: "total" } + }], + claimValues: [{ + name: "conversion_rate", + field: "conversion_rate", + unit: "%", + required: true, + tolerance: 0.000001 + }] + }] + }]); + const protocol = createDataAnalysisProtocol([], requirements); + let state = protocol.createInitialState({ contextPackageRef, runId: "run-claim-value" }); + state = reduceDataAnalysisAction(state, "data.query.plan", { + sql: "select 40 as converted, 100 as total, 0.4 as conversion_rate", + requirement_ids: ["R1"], + assertion_ids: ["R1.A1"] + }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "run_sql_readonly", { + result: { + artifact_id: "artifact-ratio", + audit_log_id: "audit-ratio", + columns: ["converted", "total", "conversion_rate"] + } + }); + state = reduceDataAnalysisAction(state, "analysis.result.validate", { + valid: true, + validation_findings: [], + verified_values: [{ + name: "conversion_rate", + value: 0.4, + unit: "%", + tolerance: 0.000001, + assertionId: "R1.A1" + }] + }); + state = reduceDataAnalysisAction(state, "analysis.evidence.bind", { + artifact_id: "artifact-ratio", + audit_log_id: "audit-ratio", + evidence_refs: ["artifact-ratio"] + }); + + expect(() => reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R1", + claim: "订单转化率为 50%", + values: [{ name: "conversion_rate", value: 0.5, unit: "%" }] + }] + })).toThrow( + 'ANALYSIS_CLAIM_VALUE_MISMATCH:R1:conversion_rate: expected value=0.4, unit="%", tolerance=0.000001; ' + + 'received value=0.5, unit="%".' + ); + expect(() => reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ requirement_id: "R1", claim: "订单转化率为 40%" }] + })).toThrow( + 'ANALYSIS_CLAIM_VALUE_REQUIRED:R1:conversion_rate: submit value=0.4, unit="%", tolerance=0.000001.' + ); + expect(() => reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R1", + claim: "提交未验证指标", + values: [{ name: "unknown_rate", value: 0.4 }] + }] + })).toThrow( + "ANALYSIS_CLAIM_VALUE_UNKNOWN:R1:unknown_rate: use an exact verified name; allowed names: conversion_rate." + ); + expect(() => reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R1", + claim: "订单转化率为 40", + values: [{ name: "conversion_rate", value: 0.4 }] + }] + })).toThrow( + 'ANALYSIS_CLAIM_VALUE_MISMATCH:R1:conversion_rate: expected value=0.4, unit="%", tolerance=0.000001; ' + + "received value=0.4, no unit." + ); + + const committed = reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R1", + claim: "订单转化率为 40%", + values: [{ name: "conversion_rate", value: 0.4, unit: "%" }] + }] + }); + expect(committed.reportedClaims[0]?.values).toEqual([ + { name: "conversion_rate", value: 0.4, unit: "%" } + ]); + }); + + it("does not report a requirement until every required assertion value has verified evidence", () => { + const multiAssertionRequirements = createUserAnalysisRequirements([{ + kind: "metric", + description: "订单数和 GMV", + acceptanceCriteria: ["返回两个指标"], + assertions: [ + { + kind: "metric", + description: "订单数", + claimValues: [{ name: "order_count", field: "order_count", required: true }] + }, + { + kind: "metric", + description: "GMV", + claimValues: [{ name: "gmv_total", field: "gmv_total", required: true }] + } + ] + }]); + let state = createDataAnalysisProtocol([], multiAssertionRequirements) + .createInitialState({ contextPackageRef, runId: "run-partial-assertion" }); + state = reduceDataAnalysisAction(state, "data.query.plan", { + sql: "select count(*) as order_count from orders", + requirement_ids: ["R1"], + assertion_ids: ["R1.A1"] + }); + state = reduceDataAnalysisAction(state, "data.query.validate", { valid: true }); + state = reduceDataAnalysisAction(state, "run_sql_readonly", { + result: { + artifact_id: "artifact-order-count", + audit_log_id: "audit-order-count", + columns: ["order_count"] + } + }); + state = reduceDataAnalysisAction(state, "analysis.result.validate", { + valid: true, + validation_findings: [], + verified_values: [{ name: "order_count", value: 3, tolerance: 0, assertionId: "R1.A1" }] + }); + state = reduceDataAnalysisAction(state, "analysis.evidence.bind", { + artifact_id: "artifact-order-count", + audit_log_id: "audit-order-count", + evidence_refs: ["artifact-order-count"] + }); + + expect(() => reduceDataAnalysisAction(state, "analysis.requirements.commit", { + claims: [{ + requirement_id: "R1", + claim: "订单数为 3", + values: [{ name: "order_count", value: 3 }] + }] + })).toThrow( + "ANALYSIS_CLAIM_VALUE_NOT_VERIFIED:R1:gmv_total: execute and bind evidence for assertion R1.A2 before committing" + ); + }); + + it("derives requirement ids from server-owned assertion ids", () => { + const requirements = createUserAnalysisRequirements([{ + kind: "metric", + description: "计算订单数", + acceptanceCriteria: ["返回订单总数"], + assertions: [{ + kind: "metric", + description: "订单总数", + sourceTables: ["orders"] + }] + }]); + const protocol = createDataAnalysisProtocol([], requirements); + const initial = protocol.createInitialState({ contextPackageRef, runId: "run-derived-requirement-ids" }); + + const planned = reduceDataAnalysisAction(initial, "data.query.plan", { + assertion_ids: ["R1.A1"], + sql: "select count(*) from orders" + }); + + expect(planned.currentQueryAttemptId).toBe("Q1"); + expect(planned.queryAttempts[0]).toMatchObject({ + requirementIds: ["R1"], + assertionIds: ["R1.A1"] + }); + }); + + it("rejects unknown requirement ids and evidence from another requirement", () => { + const requirements = createUserAnalysisRequirements([ + { kind: "metric", description: "计算利润", acceptanceCriteria: [] } + ]); + const protocol = createDataAnalysisProtocol([], requirements); + const initial = protocol.createInitialState({ contextPackageRef, runId: "run-invalid-binding" }); + + expect(() => reduceDataAnalysisAction(initial, "data.query.plan", { + requirement_ids: ["R404"], + sql: "select 1" + })).toThrow("ANALYSIS_REQUIREMENT_NOT_FOUND:R404"); + expect(() => reduceDataAnalysisAction(initial, "analysis.requirements.commit", { + claims: [{ requirement_id: "R1", claim: "profit", evidence_binding_ids: ["E404"] }] + })).toThrow("ANALYSIS_REQUIREMENT_EVIDENCE_INVALID:R1:E404"); + }); +}); diff --git a/packages/agent-runtime/src/protocol/protocols/general-task.ts b/packages/agent-runtime/src/protocol/protocols/general-task.ts new file mode 100644 index 0000000..6f3039d --- /dev/null +++ b/packages/agent-runtime/src/protocol/protocols/general-task.ts @@ -0,0 +1,70 @@ +import type { AgentProtocolDefinition } from "../types.js"; + +const DATA_ACTIONS = new Set(["list_data_sources", "inspect_schema", "preview_table", "run_sql_readonly"]); + +export type GeneralTaskState = { + answerMessageId?: string; +}; + +export const createGeneralTaskProtocol = ( + availableActionNames: string[] +): AgentProtocolDefinition => { + const workActions = [ + ...availableActionNames.filter((actionName) => !DATA_ACTIONS.has(actionName)), + "protocol.handoff.propose", + "general.answer.commit" + ]; + return { + id: "general-task", + version: "1", + initialPhase: "understand", + phases: { + understand: { + allowedActions: workActions, + transitions: [ + { targetPhase: "answer", when: ({ actionName }) => actionName === "general.answer.commit" }, + { targetPhase: "gather", when: ({ actionName }) => actionName !== "general.answer.commit" } + ] + }, + gather: { + allowedActions: workActions, + transitions: [{ + targetPhase: "answer", + when: ({ actionName }) => actionName === "general.answer.commit" + }] + }, + answer: { allowedActions: [], transitions: [] } + }, + createInitialState: () => ({}), + completionPolicy: ({ contextPackageRef, state }) => state.answerMessageId + ? { status: "completed", evaluatedContextPackageRef: contextPackageRef, evidenceRefs: [] } + : { + status: "continue", + reasons: ["GENERAL_ANSWER_NOT_COMMITTED"], + allowedActions: ["general.answer.commit"] + } + }; +}; + +export const reduceGeneralTaskAction = ( + state: GeneralTaskState, + actionName: string, + result: unknown +): GeneralTaskState => { + if (actionName !== "general.answer.commit") { + return state; + } + const messageId = recordString(result, "messageId"); + if (!messageId) { + throw new Error("GENERAL_ANSWER_MESSAGE_MISSING"); + } + return { ...state, answerMessageId: messageId }; +}; + +const recordString = (value: unknown, key: string): string | undefined => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return undefined; + } + const field = (value as Record)[key]; + return typeof field === "string" && field.length > 0 ? field : undefined; +}; diff --git a/packages/agent-runtime/src/protocol/result-verifier.test.ts b/packages/agent-runtime/src/protocol/result-verifier.test.ts new file mode 100644 index 0000000..7ce1a68 --- /dev/null +++ b/packages/agent-runtime/src/protocol/result-verifier.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; + +import { createAnalysisAssertions } from "./analysis-contract.js"; +import { verifyAnalysisResult } from "./result-verifier.js"; + +describe("analysis result verifier", () => { + it("recomputes a ratio and extracts verified claim values", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "计算负利润率", + resultChecks: [{ + kind: "ratio", + required: true, + value: { field: "negative_rate" }, + numerator: { field: "negative_orders" }, + denominator: { field: "orders" }, + tolerance: 0.000001 + }], + claimValues: [{ + name: "negative_rate", + field: "negative_rate", + required: true, + unit: "ratio", + tolerance: 0.000001 + }] + }]); + + const verification = verifyAnalysisResult({ + columns: ["orders", "negative_orders", "negative_rate"], + rows: [[200, 10, 0.05]], + rowCount: 1 + }, assertions); + + expect(verification.valid).toBe(true); + expect(verification.findings).toEqual([]); + expect(verification.verifiedValues).toEqual([{ + name: "negative_rate", + value: 0.05, + unit: "ratio", + tolerance: 0.000001, + assertionId: "R1.A1" + }]); + }); + + it("blocks verified values when a required non-empty check fails", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "必须返回订单", + resultChecks: [{ kind: "non_empty", required: true }], + claimValues: [{ name: "orders", field: "orders", required: true }] + }]); + + const verification = verifyAnalysisResult({ columns: ["orders"], rows: [], rowCount: 0 }, assertions); + + expect(verification).toMatchObject({ + valid: false, + verifiedValues: [], + findings: [{ code: "RESULT_CHECK_NON_EMPTY_FAILED", severity: "error", assertionId: "R1.A1" }] + }); + }); + + it("rejects a row count outside the declared range", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "grain", + description: "只返回三个冻结组合", + resultChecks: [{ kind: "row_count", required: true, min: 3, max: 3 }] + }]); + + const verification = verifyAnalysisResult({ columns: ["group"], rows: [["A"], ["B"]], rowCount: 2 }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_ROW_COUNT_FAILED"); + }); + + it("rejects null values in required result fields", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "利润不得为空", + resultChecks: [{ kind: "not_null", required: true, fields: ["profit"] }] + }]); + + const verification = verifyAnalysisResult({ columns: ["profit"], rows: [[null]], rowCount: 1 }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_NOT_NULL_FAILED:profit"); + }); + + it("rejects duplicate rows at the declared grain", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "grain", + description: "每个大区一行", + resultChecks: [{ kind: "unique", required: true, fields: ["region"] }] + }]); + + const verification = verifyAnalysisResult({ + columns: ["region", "orders"], + rows: [["华东", 10], ["华东", 20]], + rowCount: 2 + }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_UNIQUE_FAILED:region"); + }); + + it("resolves selectors when comparing values across result rows", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "comparison", + description: "两个方案预算相同", + resultChecks: [{ + kind: "equals", + required: true, + left: { field: "budget", selector: { plan: "A" } }, + right: { field: "budget", selector: { plan: "B" } }, + tolerance: 0.01 + }] + }]); + + const verification = verifyAnalysisResult({ + columns: ["plan", "budget"], + rows: [["A", 100], ["B", 100.02]], + rowCount: 2 + }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_EQUALS_FAILED"); + }); + + it("reconciles a total against selected part rows", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "reconciliation", + description: "分区订单数合计等于全体", + resultChecks: [{ + kind: "sum", + required: true, + total: { field: "orders", selector: { scope: "all" } }, + parts: [ + { field: "orders", selector: { scope: "east" } }, + { field: "orders", selector: { scope: "west" } } + ] + }] + }]); + + const verification = verifyAnalysisResult({ + columns: ["scope", "orders"], + rows: [["east", 40], ["west", 50], ["all", 100]], + rowCount: 3 + }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_SUM_FAILED"); + }); + + it("evaluates a declared comparison operator", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "comparison", + description: "方案A必须严格优于方案B", + resultChecks: [{ + kind: "comparison", + required: true, + left: { field: "per_10k", selector: { plan: "A" } }, + right: { field: "per_10k", selector: { plan: "B" } }, + operator: "gt" + }] + }]); + + const verification = verifyAnalysisResult({ + columns: ["plan", "per_10k"], + rows: [["A", 4.225], ["B", 4.225]], + rowCount: 2 + }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_COMPARISON_FAILED:gt"); + }); + + it("enforces counterfactual budget conservation", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "counterfactual", + description: "新增利润必须等于预算", + resultChecks: [{ + kind: "budget_conservation", + required: true, + left: { field: "budget" }, + right: { field: "incremental_profit" }, + tolerance: 0.01 + }] + }]); + + const verification = verifyAnalysisResult({ + columns: ["budget", "incremental_profit"], + rows: [[7100.6, 7099.5]], + rowCount: 1 + }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.findings[0]?.code).toBe("RESULT_CHECK_BUDGET_CONSERVATION_FAILED"); + }); + + it("rejects a missing required claim value field", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "必须暴露订单数", + claimValues: [{ name: "orders", field: "order_count", required: true }] + }]); + + const verification = verifyAnalysisResult({ columns: ["total"], rows: [[10]], rowCount: 1 }, assertions); + + expect(verification.valid).toBe(false); + expect(verification.verifiedValues).toEqual([]); + expect(verification.findings[0]?.code).toBe("RESULT_CLAIM_VALUE_MISSING:orders"); + }); +}); diff --git a/packages/agent-runtime/src/protocol/result-verifier.ts b/packages/agent-runtime/src/protocol/result-verifier.ts new file mode 100644 index 0000000..864ee65 --- /dev/null +++ b/packages/agent-runtime/src/protocol/result-verifier.ts @@ -0,0 +1,237 @@ +import type { + AnalysisAssertion, + AnalysisScalar, + AnalysisValidationFinding, + AnalysisValueOperand, + AnalysisVerifiedValue +} from "./analysis-contract.js"; + +export type AnalysisTabularResult = { + columns: string[]; + rows: unknown[]; + rowCount: number; +}; + +export type AnalysisResultVerification = { + valid: boolean; + findings: AnalysisValidationFinding[]; + verifiedValues: AnalysisVerifiedValue[]; +}; + +/** Verify deterministic result invariants and extract claimable values. */ +export const verifyAnalysisResult = ( + result: AnalysisTabularResult, + assertions: AnalysisAssertion[] +): AnalysisResultVerification => { + const rows = normalizeRows(result); + const findings: AnalysisValidationFinding[] = []; + for (const assertion of assertions) { + for (const check of assertion.resultChecks) { + if (check.kind === "non_empty") { + if (rows.length === 0) { + findings.push({ + code: "RESULT_CHECK_NON_EMPTY_FAILED", + message: `Non-empty check failed for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind === "row_count") { + const passed = (check.min === undefined || result.rowCount >= check.min) + && (check.max === undefined || result.rowCount <= check.max); + if (!passed) { + findings.push({ + code: "RESULT_CHECK_ROW_COUNT_FAILED", + message: `Row count ${result.rowCount} is outside the declared range for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind === "not_null") { + for (const field of check.fields) { + if (rows.some((row) => !Object.hasOwn(row, field) || row[field] === null)) { + findings.push({ + code: `RESULT_CHECK_NOT_NULL_FAILED:${field}`, + message: `Field ${field} contains missing or null values for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + } + continue; + } + if (check.kind === "unique") { + const keys = rows.map((row) => JSON.stringify(check.fields.map((field) => row[field]))); + if (new Set(keys).size !== keys.length) { + findings.push({ + code: `RESULT_CHECK_UNIQUE_FAILED:${check.fields.join(",")}`, + message: `Result grain is not unique for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind === "equals") { + const left = resolveOperand(check.left, rows); + const right = resolveOperand(check.right, rows); + if (!valuesEqual(left, right, check.tolerance ?? 0)) { + findings.push({ + code: "RESULT_CHECK_EQUALS_FAILED", + message: `Equality check failed for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind === "sum") { + const total = resolveOperand(check.total, rows); + const parts = check.parts.map((part) => resolveOperand(part, rows)); + const partSum = parts.every((part) => typeof part === "number") + ? parts.reduce((sum, part) => sum + (part as number), 0) + : undefined; + if (typeof total !== "number" || partSum === undefined + || Math.abs(total - partSum) > (check.tolerance ?? 0)) { + findings.push({ + code: "RESULT_CHECK_SUM_FAILED", + message: `Sum reconciliation failed for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind === "comparison") { + const left = resolveOperand(check.left, rows); + const right = resolveOperand(check.right, rows); + if (!compareValues(left, right, check.operator ?? "eq", check.tolerance ?? 0)) { + findings.push({ + code: `RESULT_CHECK_COMPARISON_FAILED:${check.operator}`, + message: `Comparison ${check.operator} failed for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind === "budget_conservation") { + const budget = resolveOperand(check.left, rows); + const conserved = resolveOperand(check.right, rows); + if (!valuesEqual(budget, conserved, check.tolerance ?? 0)) { + findings.push({ + code: "RESULT_CHECK_BUDGET_CONSERVATION_FAILED", + message: `Budget conservation failed for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + continue; + } + if (check.kind !== "ratio") continue; + const value = resolveOperand(check.value, rows); + const numerator = resolveOperand(check.numerator, rows); + const denominator = resolveOperand(check.denominator, rows); + const scale = check.scale ?? 1; + const tolerance = check.tolerance ?? 0.000001; + const passed = typeof value === "number" && typeof numerator === "number" + && typeof denominator === "number" && denominator !== 0 + && Math.abs(value - numerator / denominator * scale) <= tolerance; + if (!passed) { + findings.push({ + code: "RESULT_CHECK_RATIO_FAILED", + message: `Ratio check failed for assertion ${assertion.id}.`, + severity: check.required ? "error" : "warning", + assertionId: assertion.id + }); + } + } + } + const valid = !findings.some((finding) => finding.severity === "error"); + const verifiedValues = valid ? assertions.flatMap((assertion) => assertion.claimValues.flatMap((spec) => { + const value = resolveOperand({ + field: spec.field, + ...(spec.selector ? { selector: spec.selector } : {}) + }, rows); + if (value === undefined) { + if (spec.required) { + findings.push({ + code: `RESULT_CLAIM_VALUE_MISSING:${spec.name}`, + message: `Claim value ${spec.name} is missing for assertion ${assertion.id}.`, + severity: "error", + assertionId: assertion.id + }); + } + return []; + } + return [{ + name: spec.name, + value, + ...(spec.unit ? { unit: spec.unit } : {}), + tolerance: spec.tolerance ?? 0, + assertionId: assertion.id + }]; + })) : []; + return { + valid: valid && !findings.some((finding) => finding.severity === "error"), + findings, + verifiedValues: findings.some((finding) => finding.severity === "error") ? [] : verifiedValues + }; +}; + +const normalizeRows = (result: AnalysisTabularResult): Array> => result.rows.map((row) => { + if (isRecord(row)) { + return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, asScalar(value)])); + } + if (!Array.isArray(row)) return {}; + return Object.fromEntries(result.columns.map((column, index) => [column, asScalar(row[index])])); +}); + +const resolveOperand = ( + operand: AnalysisValueOperand, + rows: Array> +): AnalysisScalar | undefined => { + if (Object.hasOwn(operand, "literal")) return operand.literal; + if (!operand.field) return undefined; + const matchingRows = operand.selector + ? rows.filter((row) => Object.entries(operand.selector ?? {}).every(([field, value]) => row[field] === value)) + : rows; + if (matchingRows.length !== 1) return undefined; + return Object.hasOwn(matchingRows[0] as object, operand.field) + ? matchingRows[0]?.[operand.field] + : undefined; +}; + +const asScalar = (value: unknown): AnalysisScalar => + typeof value === "string" || typeof value === "number" || typeof value === "boolean" || value === null + ? value + : String(value ?? ""); + +const valuesEqual = ( + left: AnalysisScalar | undefined, + right: AnalysisScalar | undefined, + tolerance: number +): boolean => typeof left === "number" && typeof right === "number" + ? Math.abs(left - right) <= tolerance + : left !== undefined && right !== undefined && left === right; + +const compareValues = ( + left: AnalysisScalar | undefined, + right: AnalysisScalar | undefined, + operator: "eq" | "gt" | "gte" | "lt" | "lte", + tolerance: number +): boolean => { + if (operator === "eq") return valuesEqual(left, right, tolerance); + if (typeof left !== "number" || typeof right !== "number") return false; + if (operator === "gt") return left > right + tolerance; + if (operator === "gte") return left >= right - tolerance; + if (operator === "lt") return left < right - tolerance; + return left <= right + tolerance; +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/packages/agent-runtime/src/protocol/run-protocol-boundary.test.ts b/packages/agent-runtime/src/protocol/run-protocol-boundary.test.ts new file mode 100644 index 0000000..19e72d4 --- /dev/null +++ b/packages/agent-runtime/src/protocol/run-protocol-boundary.test.ts @@ -0,0 +1,1159 @@ +import { describe, expect, it } from "vitest"; + +import type { SemanticRequest, SemanticResolution } from "../semantic/types.js"; +import { ToolExecutionError } from "../errors/tool-execution-error.js"; +import { createUserAnalysisRequirements } from "./analysis-requirements.js"; +import { createRunProtocolBoundary } from "./run-protocol-boundary.js"; +import { InMemoryProtocolStateStore } from "./in-memory-protocol-state-store.js"; +import type { DataAnalysisState } from "./protocols/data-analysis.js"; +import type { ProtocolEvent } from "./types.js"; + +describe("createRunProtocolBoundary", () => { + it("extracts user requirements before starting data-analysis and skips extraction on restore", async () => { + const stateStore = new InMemoryProtocolStateStore(); + let extractionCount = 0; + const firstEvents: ProtocolEvent[] = []; + const first = await createRunProtocolBoundary({ + runId: "run-requirement-extraction", + userInput: "分析并核对利润公式", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-requirements", revision: 0 }, + tools: {}, + stateStore, + requirementExtractor: async () => { + extractionCount += 1; + return createUserAnalysisRequirements([ + { kind: "data_quality", description: "核对利润公式", acceptanceCriteria: ["报告错误数"] } + ]); + }, + projectContext: () => ({ packageId: "context-requirements", revision: 0 }), + runtimeOptions: { onEvent: (event) => firstEvents.push(event) } + }); + + expect(first.protocolRuntime.getState("run-requirement-extraction").domain).toMatchObject({ + requirements: expect.arrayContaining([expect.objectContaining({ id: "R1", description: "核对利润公式" })]) + }); + await first.dispose(); + + const restoredEvents: ProtocolEvent[] = []; + const restored = await createRunProtocolBoundary({ + runId: "run-requirement-extraction", + userInput: "继续分析", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "ignored", revision: 0 }, + tools: {}, + stateStore, + requirementExtractor: async () => { + extractionCount += 1; + throw new Error("extractor must not run during restore"); + }, + projectContext: () => ({ packageId: "context-requirements", revision: 0 }), + runtimeOptions: { onEvent: (event) => restoredEvents.push(event) } + }); + + expect(extractionCount).toBe(1); + expect(firstEvents.length).toBeGreaterThan(0); + expect(restoredEvents).toEqual([]); + expect(restored.protocolRuntime.getState("run-requirement-extraction").domain).toMatchObject({ + requirements: expect.arrayContaining([expect.objectContaining({ id: "R1" })]) + }); + }); + + it("binds SQL evidence and committed claims to extracted requirements", async () => { + const boundary = await createRunProtocolBoundary({ + runId: "run-requirement-evidence", + userInput: "分析并计算新增利润", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-requirement-evidence", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { execute: async () => ({ + result: { + artifact_id: "artifact-profit", + audit_log_id: "audit-profit", + columns: ["incremental_profit"], + rows: [[7100.6]], + row_count: 1 + } + }) } + }, + requirementExtractor: async () => createUserAnalysisRequirements([ + { kind: "metric", description: "计算新增利润", acceptanceCriteria: ["精确到分"] } + ]), + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-requirement-evidence", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-requirement-evidence", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: {} + }); + await boundary.actionRouter.execute({ + runId: "run-requirement-evidence", + segmentId: boundary.segmentId, + actionId: "sql-1", + actionName: "run_sql_readonly", + input: { + schema_id: "schema-1", + sql: "select 7100.6 as incremental_profit", + requirement_ids: ["R1"], + expected_columns: ["incremental_profit"] + } + }); + let state = boundary.protocolRuntime.getState("run-requirement-evidence"); + expect((state.domain as { requirements: Array<{ id: string; status: string }> }).requirements).toEqual( + expect.arrayContaining([expect.objectContaining({ id: "R1", status: "evidenced" })]) + ); + + await boundary.actionRouter.execute({ + runId: "run-requirement-evidence", + segmentId: boundary.segmentId, + actionId: "commit-1", + actionName: "analysis.requirements.commit", + input: { + claims: [{ + requirement_id: "R1", + claim: "新增利润为 7100.60 元", + evidence_refs: ["artifact-profit"] + }] + } + }); + state = boundary.protocolRuntime.getState("run-requirement-evidence"); + const terminal = boundary.protocolRuntime.proposeCompletion({ + runId: "run-requirement-evidence", + segmentId: boundary.segmentId, + expectedRevision: state.revision + }); + expect(terminal.terminalDecision?.status).toBe("completed"); + }); + + it("blocks evidence when a structured result invariant fails", async () => { + const boundary = await createRunProtocolBoundary({ + runId: "run-result-invariant-failure", + userInput: "分析订单转化率并核对分子分母", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-result-invariant", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1", dialect: "sqlite" }) }, + run_sql_readonly: { execute: async () => ({ + result: { + artifact_id: "artifact-ratio", + audit_log_id: "audit-ratio", + columns: ["converted", "total", "conversion_rate"], + rows: [[40, 100, 0.5]], + row_count: 1 + } + }) } + }, + requirementExtractor: async () => createUserAnalysisRequirements([{ + kind: "metric", + description: "计算订单转化率", + acceptanceCriteria: ["转化率等于转化订单数除以总订单数"], + assertions: [{ + kind: "metric", + description: "订单转化率必须可由分子分母复算", + resultChecks: [{ + kind: "ratio", + required: true, + value: { field: "conversion_rate" }, + numerator: { field: "converted" }, + denominator: { field: "total" }, + tolerance: 0.000001 + }], + claimValues: [{ + name: "conversion_rate", + field: "conversion_rate", + required: true, + tolerance: 0.000001 + }] + }] + }]), + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-result-invariant", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-result-invariant-failure", + segmentId: boundary.segmentId, + actionId: "inspect-result-invariant", + actionName: "inspect_schema", + input: {} + }); + await boundary.actionRouter.execute({ + runId: "run-result-invariant-failure", + segmentId: boundary.segmentId, + actionId: "sql-result-invariant", + actionName: "run_sql_readonly", + input: { + schema_id: "schema-1", + sql: "select 40 as converted, 100 as total, 0.5 as conversion_rate", + requirement_ids: ["R1"], + assertion_ids: ["R1.A1"], + expected_columns: ["converted", "total", "conversion_rate"] + } + }); + + const state = boundary.protocolRuntime.getState("run-result-invariant-failure").domain as DataAnalysisState; + expect(state.validationPassed).toBe(false); + expect(state.evidenceBindings).toEqual([]); + expect(state.queryAttempts[0]?.resultValidationFindings).toEqual([ + expect.objectContaining({ code: "RESULT_CHECK_RATIO_FAILED", severity: "error" }) + ]); + expect(state.queryAttempts[0]?.verifiedValues).toEqual([]); + expect(state.requirements).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: "R1", status: "queried" }) + ])); + }); + + it("routes an analytic request to data-analysis and governs every selected tool", async () => { + const boundary = await createRunProtocolBoundary({ + runId: "run-1", + userInput: "按月分析销售额", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-1", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { execute: async () => ({ + result: { + artifact_id: "artifact-1", + audit_log_id: "audit-1", + columns: ["value"], + rows: [[1]], + row_count: 1 + } + }) } + }, + semanticProvider: { + resolve: async (request) => ({ + value: { nodes: [] }, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }) + }, + semanticRequest: { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" + }, + projectContext: ({ actionName }) => ({ + contextPackageRef: { + packageId: "context-1", + revision: actionName === "inspect_schema" ? 1 : 2 + } + }) + }); + + expect(boundary.route.definition.id).toBe("data-analysis"); + expect(boundary.capabilityRegistry.resolve("inspect_schema")).toBeDefined(); + expect(boundary.capabilityRegistry.resolve("run_sql_readonly")).toBeDefined(); + + await boundary.actionRouter.execute({ + runId: "run-1", + segmentId: boundary.segmentId, + actionId: "action-1", + actionName: "inspect_schema", + input: {} + }); + expect(boundary.protocolRuntime.getState("run-1").phase).toBe("query_planning"); + + await boundary.actionRouter.execute({ + runId: "run-1", + segmentId: boundary.segmentId, + actionId: "action-2", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "select 1" } + }); + const state = boundary.protocolRuntime.getState("run-1"); + expect(state.phase).toBe("synthesis"); + expect(state.actions.map((action) => action.actionName)).toEqual([ + "inspect_schema", + "semantic.context.resolve", + "data.query.plan", + "data.query.validate", + "run_sql_readonly", + "analysis.result.validate", + "analysis.evidence.bind" + ]); + const terminal = boundary.protocolRuntime.proposeCompletion({ + runId: "run-1", + segmentId: boundary.segmentId, + expectedRevision: state.revision + }); + expect(terminal.terminalDecision?.status).toBe("completed"); + }); + + it("resolves semantic context through the configured provider before SQL execution", async () => { + const requests: unknown[] = []; + const events: ProtocolEvent[] = []; + const boundary = await createRunProtocolBoundary({ + runId: "run-semantic", + userInput: "分析订单销售额", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-semantic", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { execute: async () => ({ + result: { + artifact_id: "artifact-1", + audit_log_id: "audit-1", + columns: ["value"], + rows: [[1]], + row_count: 1 + } + }) } + }, + semanticProvider: { + resolve: async (request) => { + requests.push(request); + return { + value: { nodes: [] }, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }; + } + }, + semanticRequest: { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" + }, + runtimeOptions: { onEvent: (event) => events.push(event) }, + projectContext: () => ({ packageId: "context-semantic", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-semantic", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: { datasource_id: "orders-db" } + }); + + expect(requests).toEqual([{ + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1", + query: "分析订单销售额", + physicalSchema: { schema_id: "schema-1" } + }]); + expect(boundary.protocolRuntime.getState("run-semantic").actions.map((action) => action.actionName)).toEqual([ + "inspect_schema", + "semantic.context.resolve" + ]); + expect(boundary.protocolRuntime.getState("run-semantic").phase).toBe("query_planning"); + expect(events).toContainEqual(expect.objectContaining({ + type: "protocol.action.succeeded", + payload: { + actionId: "inspect-1:auto:1", + actionName: "semantic.context.resolve", + result: { + provider: "datalink", + mode: "live", + trust: "verified", + datasourceRevision: "schema-v1" + } + } + })); + }); + + it("grounds logical requirements after schema and semantic resolution", async () => { + const groundingInputs: unknown[] = []; + const events: ProtocolEvent[] = []; + let schemaInspectionCount = 0; + const boundary = await createRunProtocolBoundary({ + runId: "run-contract-grounding", + userInput: "统计物流单量", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-contract-grounding", revision: 0 }, + tools: { + inspect_schema: { + execute: async () => ({ + schema_id: schemaInspectionCount++ === 0 ? "schema-shipments" : "schema-shipments-refresh", + tables: [{ name: "dacomp-zh-006", columns: [{ name: "物流单号", type: "VARCHAR" }] }] + }) + } + }, + requirementExtractor: async () => createUserAnalysisRequirements([{ + kind: "metric", + description: "统计物流单量", + acceptanceCriteria: ["报告总物流单量"] + }]), + analysisContractGrounder: async (input) => { + groundingInputs.push(input); + return { + requirements: input.requirements.map((requirement) => requirement.id === "R1" + ? { + ...requirement, + assertions: [{ + id: "R1.A1", + requirementId: "R1", + kind: "metric" as const, + description: "物流单量", + required: true, + sourceTables: ["dacomp-zh-006"], + dimensions: [], + sqlConstraints: [{ + kind: "aggregate" as const, + function: "COUNT", + column: "物流单号", + alias: "shipment_count" + }], + resultChecks: [], + claimValues: [{ name: "shipment_count", field: "shipment_count", required: true }] + }] + } + : requirement), + findings: [] + }; + }, + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-contract-grounding", revision: 1 }), + runtimeOptions: { onEvent: (event) => events.push(event) } + }); + + const result = await boundary.actionRouter.execute({ + runId: "run-contract-grounding", + segmentId: boundary.segmentId, + actionId: "inspect-contract-grounding", + actionName: "inspect_schema", + input: {} + }); + + expect(groundingInputs).toEqual([expect.objectContaining({ + datasourceRevision: "schema-v1", + physicalSchema: expect.objectContaining({ schema_id: "schema-shipments" }), + semanticResolution: expect.objectContaining({ provider: "datalink", mode: "live" }) + })]); + const state = boundary.protocolRuntime.getState("run-contract-grounding"); + expect(state.phase).toBe("query_planning"); + expect(state.actions.map((action) => action.actionName)).toEqual([ + "inspect_schema", + "semantic.context.resolve", + "analysis.contract.ground" + ]); + expect(state.domain).toMatchObject({ + contractGrounded: true, + contractDatasourceRevision: "schema-v1", + requirements: expect.arrayContaining([expect.objectContaining({ + id: "R1", + assertions: [expect.objectContaining({ + id: "R1.A1", + sourceTables: ["dacomp-zh-006"] + })] + })]) + }); + expect(events).toContainEqual(expect.objectContaining({ + type: "protocol.action.succeeded", + payload: { + actionId: "inspect-contract-grounding:auto:1:auto:1", + actionName: "analysis.contract.ground", + result: { + datasourceRevision: "schema-v1", + structuredRequirementIds: ["R1"], + manualRequirementIds: [], + findings: [] + } + } + })); + expect(result.observation).toMatchObject({ + schema_id: "schema-shipments", + analysis_contract: { + requirements: [{ + requirement_id: "R1", + assertions: [{ + assertion_id: "R1.A1", + sql_constraints: [expect.objectContaining({ + kind: "aggregate", + alias: "shipment_count" + })], + claim_values: [{ name: "shipment_count", field: "shipment_count", required: true }] + }] + }] + } + }); + + await expect(boundary.actionRouter.execute({ + runId: "run-contract-grounding", + segmentId: boundary.segmentId, + actionId: "inspect-contract-grounding-again", + actionName: "inspect_schema", + input: {} + })).resolves.toBeDefined(); + expect(groundingInputs).toHaveLength(1); + expect(boundary.protocolRuntime.getState("run-contract-grounding").domain).toMatchObject({ + contractGrounded: true, + contractDatasourceRevision: "schema-v1" + }); + }); + + it("restores the latest protocol segment after an accepted handoff", async () => { + const stateStore = new InMemoryProtocolStateStore(); + const first = await createRunProtocolBoundary({ + runId: "run-handoff", + userInput: "解释这个项目", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-handoff", revision: 0 }, + tools: {}, + stateStore, + projectContext: () => ({ packageId: "context-handoff", revision: 0 }) + }); + first.handoffCoordinator.handoff({ + runId: "run-handoff", + segmentId: first.segmentId, + expectedRevision: 0, + authorizedProtocolIds: ["general-task", "data-analysis"], + target: { protocolId: "data-analysis", protocolVersion: "1" }, + reasonCodes: ["ANALYTIC_INTENT"], + unresolvedGoals: [] + }); + + const restored = await createRunProtocolBoundary({ + runId: "run-handoff", + userInput: "继续", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "ignored", revision: 0 }, + tools: {}, + stateStore, + projectContext: () => ({ packageId: "context-handoff", revision: 0 }) + }); + + expect(restored.route.definition.id).toBe("data-analysis"); + expect(restored.segmentId).toBe("run-handoff:segment:2"); + expect(restored.protocolRuntime.getState("run-handoff").status).toBe("active"); + }); + + it("rejects non-read-only SQL before the data executor runs", async () => { + let sqlExecuted = false; + const boundary = await createRunProtocolBoundary({ + runId: "run-invalid-sql", + userInput: "分析并删除订单", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-invalid", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { + execute: async () => { + sqlExecuted = true; + return {}; + } + } + }, + semanticProvider: { + resolve: async (request) => ({ + value: {}, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }) + }, + semanticRequest: { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" + }, + projectContext: () => ({ packageId: "context-invalid", revision: 1 }) + }); + await boundary.actionRouter.execute({ + runId: "run-invalid-sql", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: { datasource_id: "orders-db" } + }); + + await expect(boundary.actionRouter.execute({ + runId: "run-invalid-sql", + segmentId: boundary.segmentId, + actionId: "sql-1", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "DELETE FROM orders" } + })).rejects.toThrow("QUERY_CONTRACT_VALIDATION_FAILED"); + expect(sqlExecuted).toBe(false); + }); + + it("accepts read-only SQL with leading comments", async () => { + let sqlExecuted = false; + const boundary = await createRunProtocolBoundary({ + runId: "run-commented-sql", + userInput: "分析订单", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-commented", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { + execute: async () => { + sqlExecuted = true; + return { + result: { + artifact_id: "artifact-commented", + audit_log_id: "audit-commented", + columns: ["value"], + rows: [[1]], + row_count: 1 + } + }; + } + } + }, + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-commented", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-commented-sql", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: {} + }); + await boundary.actionRouter.execute({ + runId: "run-commented-sql", + segmentId: boundary.segmentId, + actionId: "sql-1", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "-- explain this query\nSELECT 1" } + }); + + expect(sqlExecuted).toBe(true); + }); + + it("returns SQL contract findings before an invalid query reaches the executor", async () => { + let sqlExecuted = false; + const boundary = await createRunProtocolBoundary({ + runId: "run-query-contract-failure", + userInput: "统计完整验证期订单数", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-query-contract", revision: 0 }, + tools: { + inspect_schema: { + execute: async () => ({ + schema_id: "schema-1", + dialect: "sqlite", + tables: [{ + name: "orders", + columns: [{ name: "order_date", type: "DATE" }] + }] + }) + }, + run_sql_readonly: { + execute: async () => { + sqlExecuted = true; + return { result: { rows: [], columns: [], row_count: 0 } }; + } + } + }, + requirementExtractor: async () => createUserAnalysisRequirements([{ + kind: "metric", + description: "统计完整验证期订单数", + acceptanceCriteria: ["包含12月31日"], + assertions: [{ + kind: "metric", + description: "验证期订单数", + sourceTables: ["orders"], + sqlConstraints: [{ + kind: "time_range", + column: "order_date", + start: "2023-07-01", + end: "2023-12-31", + endInclusive: true + }] + }] + }]), + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-query-contract", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-query-contract-failure", + segmentId: boundary.segmentId, + actionId: "inspect-query-contract", + actionName: "inspect_schema", + input: {} + }); + + let failure: unknown; + try { + await boundary.actionRouter.execute({ + runId: "run-query-contract-failure", + segmentId: boundary.segmentId, + actionId: "sql-query-contract", + actionName: "run_sql_readonly", + input: { + schema_id: "schema-1", + requirement_ids: ["R1"], + assertion_ids: ["R1.A1"], + sql: "SELECT COUNT(*) AS orders FROM orders WHERE order_date >= '2023-07-01'" + } + }); + } catch (error) { + failure = error; + } + + expect(sqlExecuted).toBe(false); + expect(failure).toBeInstanceOf(ToolExecutionError); + expect((failure as ToolExecutionError).observation).toMatchObject({ + error: { + code: "QUERY_CONTRACT_VALIDATION_FAILED", + executionStatus: "not_started", + message: expect.stringContaining("Required half-open end boundary 2024-01-01 is missing"), + details: { + queryAttemptId: "Q1", + findings: [expect.objectContaining({ + code: "SQL_SEMANTIC_TIME_END_MISSING:order_date", + severity: "error" + })] + } + }, + recovery: { + strategy: "refresh_and_replan", + instruction: expect.stringContaining("Required half-open end boundary 2024-01-01 is missing"), + avoid: [expect.stringContaining("same invalid SQL")] + } + }); + const state = boundary.protocolRuntime.getState("run-query-contract-failure"); + expect(state.phase).toBe("query_planning"); + expect(state.actions.map((action) => action.actionName)).toEqual([ + "inspect_schema", + "semantic.context.resolve", + "analysis.contract.ground", + "data.query.plan", + "data.query.validate" + ]); + }); + + it.each([ + "report.md", + "report.markdown", + "report.txt", + "report.html", + "report.md/ " + ])("requires evidenced claims to be committed before writing synthesis output %s", async (reportPath) => { + let reportWrites = 0; + const boundary = await createRunProtocolBoundary({ + runId: "run-report-commit-order", + userInput: "计算订单数并输出 Markdown 报告", + authorizedProtocolIds: ["general-task", "data-analysis"], + explicitProtocol: { protocolId: "data-analysis", protocolVersion: "1" }, + initialContextPackageRef: { packageId: "context-report-order", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { execute: async () => ({ + result: { + artifact_id: "artifact-orders", + audit_log_id: "audit-orders", + columns: ["order_count"], + rows: [[10]], + row_count: 1 + } + }) }, + write_file: { + execute: async () => { + reportWrites += 1; + return { ok: true }; + } + } + }, + requirementExtractor: async () => createUserAnalysisRequirements([{ + kind: "metric", + description: "计算订单数", + acceptanceCriteria: ["报告订单总数"] + }]), + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-report-order", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-report-commit-order", + segmentId: boundary.segmentId, + actionId: "inspect-report-order", + actionName: "inspect_schema", + input: {} + }); + await boundary.actionRouter.execute({ + runId: "run-report-commit-order", + segmentId: boundary.segmentId, + actionId: "sql-report-order", + actionName: "run_sql_readonly", + input: { + schema_id: "schema-1", + requirement_ids: ["R1"], + sql: "select 10 as order_count" + } + }); + + await expect(boundary.actionRouter.execute({ + runId: "run-report-commit-order", + segmentId: boundary.segmentId, + actionId: "write-report-too-early", + actionName: "write_file", + input: { path: reportPath, content: "# report" } + })).rejects.toMatchObject({ + observation: { + error: { + code: "ANALYSIS_REQUIREMENTS_COMMIT_REQUIRED", + details: { requirementIds: ["R1"] } + } + } + }); + expect(reportWrites).toBe(0); + + await boundary.actionRouter.execute({ + runId: "run-report-commit-order", + segmentId: boundary.segmentId, + actionId: "commit-report-order", + actionName: "analysis.requirements.commit", + input: { claims: [{ requirement_id: "R1", claim: "订单数为 10" }] } + }); + await boundary.actionRouter.execute({ + runId: "run-report-commit-order", + segmentId: boundary.segmentId, + actionId: "write-report-after-commit", + actionName: "write_file", + input: { path: reportPath, content: "# report" } + }); + expect(reportWrites).toBe(1); + }); + + it("supports more than one hundred governed actions in a complex data run", async () => { + const boundary = await createRunProtocolBoundary({ + runId: "run-complex-budget", + userInput: "执行复杂多轮分析", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-budget", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { execute: async () => ({ + result: { + artifact_id: "artifact-budget", + audit_log_id: "audit-budget", + columns: ["value"], + rows: [[1]], + row_count: 1 + } + }) } + }, + semanticProvider: liveSemanticProvider(), + semanticRequest: semanticRequest(), + projectContext: () => ({ packageId: "context-budget", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-complex-budget", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: {} + }); + for (let index = 0; index < 21; index += 1) { + await boundary.actionRouter.execute({ + runId: "run-complex-budget", + segmentId: boundary.segmentId, + actionId: `sql-${index + 1}`, + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: `SELECT ${index + 1}` } + }); + } + + expect(boundary.protocolRuntime.getState("run-complex-budget").actions.length).toBeGreaterThan(100); + }); + + it("recovers after a SQL execution failure and completes a later query attempt", async () => { + let sqlCalls = 0; + const boundary = await createRunProtocolBoundary({ + runId: "run-sql-recovery", + userInput: "分析订单并在查询失败后重试", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-recovery", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + preview_table: { execute: async () => ({ rows: [[1]] }) }, + run_sql_readonly: { + execute: async () => { + sqlCalls += 1; + if (sqlCalls === 2) { + throw new Error("no such column: missing_column"); + } + return { + result: { + artifact_id: `artifact-${sqlCalls}`, + audit_log_id: `audit-${sqlCalls}`, + columns: ["value"], + rows: [[sqlCalls]], + row_count: 1 + } + }; + } + } + }, + semanticProvider: { + resolve: async (request) => ({ + value: {}, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }) + }, + semanticRequest: { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" + }, + projectContext: () => ({ packageId: "context-recovery", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: { datasource_id: "orders-db" } + }); + await boundary.actionRouter.execute({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + actionId: "preview-1", + actionName: "preview_table", + input: { schema_id: "schema-1", table: "orders" } + }); + await boundary.actionRouter.execute({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + actionId: "sql-1", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "select 1" } + }); + await expect(boundary.actionRouter.execute({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + actionId: "sql-2", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "select missing_column" } + })).rejects.toThrow("no such column: missing_column"); + + expect(boundary.protocolRuntime.getState("run-sql-recovery")).toMatchObject({ + phase: "execution", + domain: { queryExecuted: false, validationPassed: false } + }); + await boundary.actionRouter.execute({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + actionId: "inspect-2", + actionName: "inspect_schema", + input: { datasource_id: "orders-db" } + }); + await boundary.actionRouter.execute({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + actionId: "sql-3", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "select 3" } + }); + + const state = boundary.protocolRuntime.getState("run-sql-recovery"); + expect(state.phase).toBe("synthesis"); + expect(state.actions.filter((action) => action.status === "failed")).toHaveLength(1); + const terminal = boundary.protocolRuntime.proposeCompletion({ + runId: "run-sql-recovery", + segmentId: boundary.segmentId, + expectedRevision: state.revision + }); + expect(terminal.terminalDecision?.status).toBe("completed"); + }); + + it("does not validate a SQL result that lacks tabular audit evidence", async () => { + const boundary = await createRunProtocolBoundary({ + runId: "run-invalid-result", + userInput: "分析订单", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-invalid-result", revision: 0 }, + tools: { + inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) }, + run_sql_readonly: { execute: async () => ({ result: { artifact_id: "artifact-without-rows" } }) } + }, + semanticProvider: { + resolve: async (request) => ({ + value: {}, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }) + }, + semanticRequest: { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" + }, + projectContext: () => ({ packageId: "context-invalid-result", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-invalid-result", + segmentId: boundary.segmentId, + actionId: "inspect-1", + actionName: "inspect_schema", + input: {} + }); + await boundary.actionRouter.execute({ + runId: "run-invalid-result", + segmentId: boundary.segmentId, + actionId: "sql-1", + actionName: "run_sql_readonly", + input: { schema_id: "schema-1", sql: "select 1" } + }); + + const state = boundary.protocolRuntime.getState("run-invalid-result"); + expect(state).toMatchObject({ phase: "validation", domain: { validationPassed: false } }); + const completion = boundary.protocolRuntime.proposeCompletion({ + runId: "run-invalid-result", + segmentId: boundary.segmentId, + expectedRevision: state.revision + }); + expect(completion.terminalDecision).toBeUndefined(); + }); + + it("journals routing events before protocol segment start", async () => { + const eventTypes: string[] = []; + + await createRunProtocolBoundary({ + runId: "run-route-events", + userInput: "你好", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "context-route", revision: 0 }, + tools: {}, + projectContext: () => ({ packageId: "context-route", revision: 0 }), + runtimeOptions: { onEvent: (event) => eventTypes.push(event.type) } + }); + + expect(eventTypes.slice(0, 4)).toEqual([ + "protocol.route.requested", + "protocol.route.resolved", + "protocol.run.started", + "protocol.phase.entered" + ]); + }); + + it("emits a routing failure before model assembly is allowed to continue", async () => { + const eventTypes: string[] = []; + + await expect(createRunProtocolBoundary({ + runId: "run-route-failed", + userInput: "分析数据", + authorizedProtocolIds: ["general-task"], + explicitProtocol: { protocolId: "data-analysis", protocolVersion: "1" }, + initialContextPackageRef: { packageId: "context-route", revision: 0 }, + tools: {}, + projectContext: () => ({ packageId: "context-route", revision: 0 }), + runtimeOptions: { onEvent: (event) => eventTypes.push(event.type) } + })).rejects.toThrow("PROTOCOL_NOT_AUTHORIZED:data-analysis@1"); + + expect(eventTypes).toEqual(["protocol.route.failed"]); + }); + + it("switches the active runtime after an Agent handoff proposal is accepted", async () => { + const boundary = await createRunProtocolBoundary({ + runId: "run-agent-handoff", + userInput: "先解释，随后分析", + authorizedProtocolIds: ["general-task", "data-analysis"], + explicitProtocol: { protocolId: "general-task", protocolVersion: "1" }, + initialContextPackageRef: { packageId: "context-handoff", revision: 0 }, + tools: { inspect_schema: { execute: async () => ({ schema_id: "schema-1" }) } }, + semanticProvider: { + resolve: async (request) => ({ + value: {}, + capabilities: ["graph-explore"], + trust: "verified", + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }) + }, + semanticRequest: { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" + }, + projectContext: () => ({ packageId: "context-handoff", revision: 1 }) + }); + + await boundary.actionRouter.execute({ + runId: "run-agent-handoff", + segmentId: boundary.segmentId, + actionId: "handoff-1", + actionName: "protocol.handoff.propose", + input: { + targetProtocolId: "data-analysis", + targetProtocolVersion: "1", + reasonCodes: ["ANALYTIC_INTENT"], + unresolvedGoals: [] + } + }); + + expect(boundary.segmentId).toBe("run-agent-handoff:segment:2"); + expect(boundary.protocolRuntime.getState("run-agent-handoff")).toMatchObject({ + protocolId: "data-analysis", + phase: "scope", + status: "active" + }); + await boundary.actionRouter.execute({ + runId: "run-agent-handoff", + segmentId: boundary.segmentId, + actionId: "inspect-after-handoff", + actionName: "inspect_schema", + input: {} + }); + expect(boundary.protocolRuntime.getState("run-agent-handoff").phase).toBe("query_planning"); + }); +}); + +const liveSemanticProvider = (): { resolve(request: SemanticRequest): Promise } => ({ + resolve: async (request: SemanticRequest) => ({ + value: {}, + capabilities: ["graph-explore"], + trust: "verified" as const, + warnings: [], + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }) +}); + +const semanticRequest = () => ({ + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1" +}); diff --git a/packages/agent-runtime/src/protocol/run-protocol-boundary.ts b/packages/agent-runtime/src/protocol/run-protocol-boundary.ts new file mode 100644 index 0000000..601b067 --- /dev/null +++ b/packages/agent-runtime/src/protocol/run-protocol-boundary.ts @@ -0,0 +1,729 @@ +import { z } from "zod"; +import { ToolExecutionError } from "../errors/tool-execution-error.js"; +import { AGENT_RUNTIME_LIMITS } from "../config/agent-runtime-limits.js"; + +import { + ActionRouter, + type ActionContextProjection, + type ActionRouterOptions +} from "../capabilities/action-router.js"; +import { CapabilityRegistry } from "../capabilities/capability-registry.js"; +import { createToolCapabilityPlugin } from "../capabilities/tool-capability-plugin.js"; +import type { CapabilityPlugin } from "../capabilities/types.js"; +import type { AnalysisRequirementExtractor } from "./model-analysis-requirement-extractor.js"; +import type { + AnalysisContractGrounder, + AnalysisContractGroundingInput +} from "./model-analysis-contract-grounder.js"; +import type { AnalysisValidationFinding } from "./analysis-contract.js"; +import { InMemoryProtocolStateStore } from "./in-memory-protocol-state-store.js"; +import { ProtocolHandoffCoordinator } from "./protocol-handoff-coordinator.js"; +import { ProtocolRegistry } from "./protocol-registry.js"; +import { + ProtocolRouter, + type ProtocolClassifier, + type ProtocolIdentity, + type ProtocolRouteResult +} from "./protocol-router.js"; +import { ProtocolRuntime, type ProtocolRuntimeOptions } from "./protocol-runtime.js"; +import { verifyAnalysisResult } from "./result-verifier.js"; +import { + createDataAnalysisProtocol, + reduceDataAnalysisAction, + type DataAnalysisState +} from "./protocols/data-analysis.js"; +import { + createGeneralTaskProtocol, + reduceGeneralTaskAction, + type GeneralTaskState +} from "./protocols/general-task.js"; +import type { + AgentProtocolDefinition, + ContextPackageRef, + ProtocolGuardResult, + ProtocolStateStore +} from "./types.js"; +import type { SemanticRequest, SemanticResolution } from "../semantic/types.js"; + +type ExistingTool = { execute?: (...args: unknown[]) => unknown | Promise }; +type RunProtocolDomainState = GeneralTaskState | DataAnalysisState; + +export type CreateRunProtocolBoundaryInput = { + runId: string; + userInput: string; + authorizedProtocolIds: string[]; + initialContextPackageRef: ContextPackageRef; + tools: Record; + explicitProtocol?: ProtocolIdentity; + classifier?: ProtocolClassifier; + projectContext: ActionRouterOptions["projectContext"]; + serverPolicy?: ActionRouterOptions["serverPolicy"]; + resourceAuthorization?: ActionRouterOptions["resourceAuthorization"]; + runtimeOptions?: ProtocolRuntimeOptions; + stateStore?: ProtocolStateStore; + semanticProvider?: { resolve(request: SemanticRequest): Promise }; + semanticRequest?: Omit; + requirementExtractor?: AnalysisRequirementExtractor; + analysisContractGrounder?: AnalysisContractGrounder; +}; + +export type RunProtocolBoundary = { + actionRouter: ActionRouter; + capabilityRegistry: CapabilityRegistry; + protocolRuntime: ProtocolRuntime; + handoffCoordinator: ProtocolHandoffCoordinator; + route: ProtocolRouteResult; + segmentId: string; + acknowledgeEvent(event: import("./types.js").ProtocolEvent): void; + dispose(): Promise; +}; + +/** Resolve a formal protocol and bind every selected tool to its governed action boundary. */ +export const createRunProtocolBoundary = async ( + input: CreateRunProtocolBoundaryInput +): Promise => { + const actionNames = Object.keys(input.tools); + const stateStore = input.stateStore ?? new InMemoryProtocolStateStore(); + const persistedState = stateStore.find(input.runId); + if (persistedState && !input.authorizedProtocolIds.includes(persistedState.protocolId)) { + throw new Error(`PROTOCOL_NOT_AUTHORIZED:${persistedState.protocolId}@${persistedState.protocolVersion}`); + } + if ( + persistedState + && input.explicitProtocol + && (input.explicitProtocol.protocolId !== persistedState.protocolId + || input.explicitProtocol.protocolVersion !== persistedState.protocolVersion) + ) { + throw new Error("PROTOCOL_RESUME_SELECTION_MISMATCH"); + } + const shouldExtractRequirements = !persistedState + && Boolean(input.requirementExtractor) + && (input.explicitProtocol?.protocolId === "data-analysis" || analyticIntent(input.userInput)); + const userRequirements = shouldExtractRequirements + ? await input.requirementExtractor?.({ userText: input.userInput }) ?? [] + : []; + const protocolRegistry = new ProtocolRegistry(); + protocolRegistry.register(createGeneralTaskProtocol(actionNames)); + protocolRegistry.register(createDataAnalysisProtocol(actionNames, userRequirements)); + const router = new ProtocolRouter(protocolRegistry, { + ...(input.classifier ? { classifier: input.classifier } : {}) + }); + let route: ProtocolRouteResult; + try { + route = await router.route({ + authorizedProtocolIds: input.authorizedProtocolIds, + ...(!persistedState && input.explicitProtocol ? { explicit: input.explicitProtocol } : {}), + deterministicCandidates: persistedState + ? [{ + protocolId: persistedState.protocolId, + protocolVersion: persistedState.protocolVersion, + priority: 1000, + reasonCode: "PROTOCOL_SEGMENT_RESTORED" + }] + : analyticIntent(input.userInput) + ? [{ + protocolId: "data-analysis", + protocolVersion: "1", + priority: 100, + reasonCode: "ANALYTIC_INTENT" + }] + : [], + classificationInput: { userText: input.userInput } + }); + } catch (error) { + input.runtimeOptions?.onEvent?.({ + eventId: `${input.runId}:segment:1:0:protocol.route.failed`, + type: "protocol.route.failed", + runId: input.runId, + segmentId: `${input.runId}:segment:1`, + protocolId: "unresolved", + protocolVersion: "0", + revision: 0, + payload: { reason: error instanceof Error ? error.message : String(error) } + }); + throw error; + } + let activeProtocolId = route.definition.id; + const reduceAction = (state: unknown, actionName: string, result: unknown): unknown => + activeProtocolId === "data-analysis" + ? reduceDataAnalysisAction(state as DataAnalysisState, actionName, result) + : reduceGeneralTaskAction(state as GeneralTaskState, actionName, result); + const capabilityRegistry = new CapabilityRegistry(); + capabilityRegistry.register(createToolCapabilityPlugin({ + id: "selected-run-tools", + tools: input.tools, + reduceAction + })); + capabilityRegistry.register(createRuntimeActionPlugin( + reduceAction, + input.semanticProvider, + input.analysisContractGrounder + )); + await capabilityRegistry.initialize(); + let segmentId = persistedState?.segmentId ?? `${input.runId}:segment:1`; + const runtimeOptions: ProtocolRuntimeOptions = { + ...(input.runtimeOptions ?? {}), + maxActions: input.runtimeOptions?.maxActions ?? (route.definition.id === "data-analysis" + ? AGENT_RUNTIME_LIMITS.dataAnalysisMaxProtocolActions + : AGENT_RUNTIME_LIMITS.generalTaskMaxProtocolActions), + ...(!persistedState + ? { + startEvents: [ + { + type: "protocol.route.requested", + payload: { + authorizedProtocolIds: input.authorizedProtocolIds, + explicit: input.explicitProtocol + } + }, + ...(route.source === "classifier" + ? [{ + type: "protocol.route.classified", + payload: { reasonCodes: route.reasonCodes, warnings: route.warnings } + }] + : []), + { + type: "protocol.route.resolved", + payload: { + protocolId: route.definition.id, + protocolVersion: route.definition.version, + reasonCodes: route.reasonCodes, + source: route.source, + warnings: route.warnings + } + }, + ...(route.definition.id === "data-analysis" && userRequirements.length > 0 + ? [{ + type: "analysis.requirements.extracted", + payload: { + requirements: userRequirements.map((requirement) => ({ + id: requirement.id, + kind: requirement.kind, + description: requirement.description, + required: requirement.required, + assertions: requirement.assertions.map((assertion) => ({ + id: assertion.id, + kind: assertion.kind, + required: assertion.required + })) + })) + } + }] + : []) + ] + } + : {}) + }; + let protocolRuntime = new ProtocolRuntime( + route.definition as AgentProtocolDefinition, + stateStore, + runtimeOptions + ); + if (persistedState) { + protocolRuntime.restore(input.runId, segmentId); + } else { + protocolRuntime.start({ + runId: input.runId, + segmentId, + contextPackageRef: input.initialContextPackageRef + }); + } + const handoffCoordinator = new ProtocolHandoffCoordinator(protocolRegistry, stateStore, { + ...(runtimeOptions.onEvent ? { onEvent: runtimeOptions.onEvent } : {}) + }); + const actionRouter = new ActionRouter(capabilityRegistry, protocolRuntime, { + automaticActions: (actionInput) => activeProtocolId === "data-analysis" + ? dataAnalysisAutomaticActions(actionInput, input) + : [], + preparatoryActions: (actionInput) => activeProtocolId === "data-analysis" + ? dataAnalysisPreparatoryActions(actionInput) + : [], + afterPreparatoryActions: ({ actionName, domain, input: actionInput, phase }) => { + if (activeProtocolId !== "data-analysis") { + return; + } + const dataAnalysisState = domain as DataAnalysisState; + if (actionName === "run_sql_readonly") { + assertCurrentQueryContract(dataAnalysisState); + } + if (isReportFileAction(actionName, actionInput, phase)) { + assertRequirementsCommittedBeforeReport(dataAnalysisState); + } + }, + serverPolicy: input.serverPolicy ?? allowAction, + ...(input.resourceAuthorization ? { resourceAuthorization: input.resourceAuthorization } : {}), + projectContext: input.projectContext, + projectFinalObservation: ({ actionName, domain, observation }) => + activeProtocolId === "data-analysis" && actionName === "inspect_schema" + ? projectGroundedSchemaObservation(observation, domain as DataAnalysisState) + : observation, + projectProtocolEventResult: ({ actionName, rawResult }) => { + if (actionName === "semantic.context.resolve") { + return semanticResolutionEventResult(rawResult); + } + return actionName === "analysis.contract.ground" + ? analysisContractGroundingEventResult(rawResult) + : undefined; + }, + afterAction: ({ actionName, rawResult }) => { + if (actionName !== "protocol.handoff.propose") { + return; + } + const targetProtocolId = directString(rawResult, "targetProtocolId"); + const targetProtocolVersion = directString(rawResult, "targetProtocolVersion"); + if (!targetProtocolId || !targetProtocolVersion) { + throw new Error("PROTOCOL_HANDOFF_PROPOSAL_INVALID"); + } + const current = protocolRuntime.getState(input.runId, segmentId); + const handoff = handoffCoordinator.handoff({ + runId: input.runId, + segmentId, + expectedRevision: current.revision, + authorizedProtocolIds: input.authorizedProtocolIds, + target: { protocolId: targetProtocolId, protocolVersion: targetProtocolVersion }, + reasonCodes: recordStringArray(rawResult, "reasonCodes"), + unresolvedGoals: recordStringArray(rawResult, "unresolvedGoals") + }); + const targetDefinition = protocolRegistry.find(targetProtocolId, targetProtocolVersion); + if (!targetDefinition) { + throw new Error("PROTOCOL_HANDOFF_TARGET_UNAVAILABLE"); + } + activeProtocolId = targetProtocolId; + segmentId = handoff.next.segmentId; + protocolRuntime = new ProtocolRuntime( + targetDefinition as AgentProtocolDefinition, + stateStore, + { ...runtimeOptions, startEvents: [] } + ); + protocolRuntime.restore(input.runId, segmentId); + actionRouter.replaceProtocolRuntime(protocolRuntime); + } + }); + return { + actionRouter, + capabilityRegistry, + handoffCoordinator, + route, + get protocolRuntime() { + return protocolRuntime; + }, + get segmentId() { + return segmentId; + }, + acknowledgeEvent: (event) => stateStore.acknowledgeEvent(event), + dispose: () => capabilityRegistry.dispose() + }; +}; + +const semanticResolutionEventResult = (value: unknown): Record => { + const provider = directString(value, "provider"); + const mode = directString(value, "mode"); + const trust = directString(value, "trust"); + const datasourceRevision = directString(value, "datasourceRevision"); + const fallbackReason = directString(value, "fallbackReason"); + return { + ...(provider ? { provider } : {}), + ...(mode ? { mode } : {}), + ...(trust ? { trust } : {}), + ...(datasourceRevision ? { datasourceRevision } : {}), + ...(fallbackReason ? { fallbackReason } : {}) + }; +}; + +const analysisContractGroundingEventResult = (value: unknown): Record => { + const requirements = recordArray(value, "requirements").filter((requirement) => + directString(requirement, "source") === "user"); + const structuredRequirementIds: string[] = []; + const manualRequirementIds: string[] = []; + for (const requirement of requirements) { + const requirementId = directString(requirement, "id"); + if (!requirementId) { + continue; + } + const hasStructuredAssertion = recordArray(requirement, "assertions").some((assertion) => + directString(assertion, "kind") !== "manual"); + (hasStructuredAssertion ? structuredRequirementIds : manualRequirementIds).push(requirementId); + } + return { + ...(directString(value, "datasourceRevision") + ? { datasourceRevision: directString(value, "datasourceRevision") } + : {}), + structuredRequirementIds, + manualRequirementIds, + findings: recordArray(value, "findings").map((finding) => ({ + ...(directString(finding, "requirementId") + ? { requirementId: directString(finding, "requirementId") } + : {}), + ...(directString(finding, "code") ? { code: directString(finding, "code") } : {}), + ...(directString(finding, "message") ? { message: directString(finding, "message") } : {}) + })) + }; +}; + +const projectGroundedSchemaObservation = ( + observation: unknown, + state: DataAnalysisState +): Record => { + const schemaObservation = typeof observation === "object" && observation !== null && !Array.isArray(observation) + ? observation as Record + : { schema: observation }; + return { + ...schemaObservation, + analysis_contract: { + instruction: [ + "Use the exact requirement_id, assertion_id, aggregate aliases, and expected columns below in", + "run_sql_readonly. Do not invent or rename contract fields." + ].join(" "), + requirements: state.requirements + .filter((requirement) => requirement.source === "user") + .map((requirement) => ({ + requirement_id: requirement.id, + description: requirement.description, + acceptance_criteria: [...requirement.acceptanceCriteria], + assertions: requirement.assertions.map((assertion) => ({ + assertion_id: assertion.id, + kind: assertion.kind, + description: assertion.description, + source_tables: [...assertion.sourceTables], + dimensions: [...assertion.dimensions], + sql_constraints: structuredClone(assertion.sqlConstraints), + result_checks: structuredClone(assertion.resultChecks), + claim_values: structuredClone(assertion.claimValues) + })) + })) + } + }; +}; + +const createRuntimeActionPlugin = ( + reduceAction: (state: unknown, actionName: string, result: unknown) => unknown, + semanticProvider?: { resolve(request: SemanticRequest): Promise }, + analysisContractGrounder?: AnalysisContractGrounder +): CapabilityPlugin => { + const names = [ + "general.answer.commit", + "protocol.handoff.propose", + "semantic.context.resolve", + "analysis.contract.ground", + "data.query.plan", + "data.query.validate", + "analysis.result.validate", + "analysis.evidence.bind", + "analysis.requirements.commit" + ]; + return { + manifest: { id: "protocol-runtime-actions", version: "1", provides: names }, + actions: names.map((name) => ({ + name, + exposure: name === "protocol.handoff.propose" || name === "analysis.requirements.commit" ? "agent" : "runtime", + inputSchema: z.unknown(), + outputSchema: z.unknown(), + idempotency: "supported", + execute: async (_context, actionInput) => executeRuntimeAction( + name, + actionInput, + semanticProvider, + analysisContractGrounder + ), + reduce: (state, result) => reduceAction(state, name, result) + })) + }; +}; + +const executeRuntimeAction = async ( + name: string, + actionInput: unknown, + semanticProvider?: { resolve(request: SemanticRequest): Promise }, + analysisContractGrounder?: AnalysisContractGrounder +): Promise => { + if (name === "semantic.context.resolve" && semanticProvider) { + return semanticProvider.resolve(actionInput as SemanticRequest); + } + if (name === "analysis.contract.ground") { + const groundingInput = actionInput as AnalysisContractGroundingInput; + const result = analysisContractGrounder + ? await analysisContractGrounder(groundingInput) + : { requirements: groundingInput.requirements, findings: [] }; + return { + ...result, + datasourceRevision: groundingInput.datasourceRevision, + schema_id: directString(groundingInput.physicalSchema, "schema_id") + ?? directString(groundingInput.physicalSchema, "schemaId") + }; + } + if (name === "data.query.validate") { + const sql = directString(actionInput, "sql"); + const schemaId = directString(actionInput, "schema_id") ?? directString(actionInput, "schemaId"); + const sqlReasons = validateReadonlySql(sql); + return { + valid: Boolean(sql && schemaId && sqlReasons.length === 0), + reasons: [ + ...(sql ? [] : ["SQL_REQUIRED"]), + ...(schemaId ? [] : ["SCHEMA_ID_REQUIRED"]), + ...sqlReasons + ] + }; + } + return actionInput; +}; + +const validateReadonlySql = (sql: string | undefined): string[] => { + if (!sql) { + return []; + } + const normalized = stripLeadingSqlComments(sql).trim().replace(/;\s*$/u, ""); + const reasons: string[] = []; + if (!/^(?:select|with)\b/iu.test(normalized)) { + reasons.push("SQL_NOT_READ_ONLY"); + } + if ( + /\b(?:insert|update|delete|drop|alter|create|grant|revoke|copy|call|pragma|attach|detach|vacuum|truncate|merge)\b/iu + .test(normalized) + ) { + reasons.push("SQL_MUTATION_KEYWORD_FORBIDDEN"); + } + if (normalized.includes(";")) { + reasons.push("SQL_MULTIPLE_STATEMENTS_FORBIDDEN"); + } + return reasons; +}; + +const stripLeadingSqlComments = (sql: string): string => { + let remaining = sql.trimStart(); + while (remaining.startsWith("--") || remaining.startsWith("/*")) { + if (remaining.startsWith("--")) { + const lineEnd = remaining.indexOf("\n"); + remaining = lineEnd < 0 ? "" : remaining.slice(lineEnd + 1).trimStart(); + continue; + } + const blockEnd = remaining.indexOf("*/", 2); + remaining = blockEnd < 0 ? "" : remaining.slice(blockEnd + 2).trimStart(); + } + return remaining; +}; + +const analyticIntent = (userInput: string): boolean => + /\b(?:sql|query|metric|analytics?|statistics?)\b|分析|统计|指标|数据|销售额/iu.test(userInput); + +const allowAction = (): ProtocolGuardResult => ({ allowed: true }); + +const dataAnalysisPreparatoryActions = (input: { + actionName: string; + input: unknown; +}): Array<{ actionName: string; input: unknown }> => input.actionName === "run_sql_readonly" + ? [ + { actionName: "data.query.plan", input: input.input }, + { actionName: "data.query.validate", input: input.input } + ] + : []; + +const assertCurrentQueryContract = (state: DataAnalysisState): void => { + if (state.currentQueryValidated) { + return; + } + const attempt = state.queryAttempts.find((candidate) => candidate.id === state.currentQueryAttemptId) + ?? state.queryAttempts.at(-1); + const findings = attempt?.validationFindings ?? []; + const findingCodes = findings.map((finding) => finding.code).join(", ") || "QUERY_CONTRACT_INVALID"; + const exactCorrections = findings.map((finding) => finding.message).join(" ") + || "The query does not satisfy its selected analysis assertions."; + throw new ToolExecutionError({ + ok: false, + isError: true, + error: { + code: "QUERY_CONTRACT_VALIDATION_FAILED", + category: "validation", + message: `SQL was not executed because contract validation failed: ${findingCodes}. ${exactCorrections}`, + executionStatus: "not_started", + retryable: false, + details: { + queryAttemptId: attempt?.id ?? "unknown", + findings: structuredClone(findings), + allowedActions: ["data.query.plan", "data.query.validate", "inspect_schema", "preview_table"] + } + }, + recovery: { + strategy: "refresh_and_replan", + instruction: `Apply these exact SQL corrections, then submit a new query plan: ${exactCorrections}`, + avoid: ["Do not repeat the same invalid SQL without addressing the listed findings."] + } + }); +}; + +const assertRequirementsCommittedBeforeReport = (state: DataAnalysisState): void => { + const incomplete = state.requirements.filter((requirement) => + requirement.source === "user" && requirement.required && requirement.status !== "reported"); + if (incomplete.length === 0) { + return; + } + throw new ToolExecutionError({ + ok: false, + isError: true, + error: { + code: "ANALYSIS_REQUIREMENTS_COMMIT_REQUIRED", + category: "validation", + message: "The final analysis output cannot be written until every required analysis claim is committed.", + executionStatus: "not_started", + retryable: false, + details: { + requirementIds: incomplete.map((requirement) => requirement.id), + requirements: incomplete.map((requirement) => ({ + id: requirement.id, + status: requirement.status, + recovery: requirement.status === "evidenced" + ? "Commit this claim with analysis_requirements_commit." + : "Finish validated SQL evidence before committing this claim." + })) + } + }, + recovery: { + strategy: "refresh_and_replan", + instruction: "Commit evidenced claims, finish any still-pending analyses, then write the final output.", + avoid: ["Do not retry the final report write while required claims remain unreported."] + } + }); +}; + +const isReportFileAction = (actionName: string, input: unknown, phase: string): boolean => { + if (actionName !== "write_file" && actionName !== "edit_file") { + return false; + } + if (phase === "synthesis") { + return true; + } + const filePath = directString(input, "path") ?? directString(input, "filename") ?? ""; + return /\.(?:html?|markdown|md|rst|txt)$/iu.test(filePath.trim().replace(/\/+$/u, "")); +}; + +const dataAnalysisAutomaticActions = (input: { + actionName: string; + domain: unknown; + input: unknown; + rawResult: unknown; +}, boundaryInput: CreateRunProtocolBoundaryInput): Array<{ actionName: string; input: unknown }> => { + if (input.actionName === "inspect_schema" && boundaryInput.semanticProvider && boundaryInput.semanticRequest) { + return [{ + actionName: "semantic.context.resolve", + input: { + ...boundaryInput.semanticRequest, + query: boundaryInput.userInput, + physicalSchema: input.rawResult + } + }]; + } + if (input.actionName === "semantic.context.resolve") { + const state = input.domain as DataAnalysisState; + const userRequirements = state.requirements.filter((requirement) => requirement.source === "user"); + if (userRequirements.length === 0 || state.contractGrounded) { + return []; + } + return [{ + actionName: "analysis.contract.ground", + input: { + requirements: state.requirements, + physicalSchema: recordValue(input.input, "physicalSchema"), + semanticResolution: input.rawResult, + datasourceRevision: directString(input.input, "datasourceRevision") ?? "unknown" + } + }]; + } + if (input.actionName !== "run_sql_readonly") { + return []; + } + const artifactId = nestedString(input.rawResult, "result", "artifact_id") + ?? directString(input.rawResult, "artifact_id"); + const auditLogId = nestedString(input.rawResult, "result", "audit_log_id") + ?? directString(input.rawResult, "audit_log_id"); + const resultFields = nestedStringArray(input.rawResult, "result", "columns"); + const validation = validateAnalysisResult(input.rawResult, input.input, input.domain as DataAnalysisState); + return [ + { actionName: "analysis.result.validate", input: validation }, + ...(artifactId && validation.valid + ? [{ + actionName: "analysis.evidence.bind", + input: { + artifact_id: artifactId, + ...(auditLogId ? { audit_log_id: auditLogId } : {}), + evidence_refs: [artifactId], + result_fields: resultFields + } + }] + : []) + ]; +}; + +const validateAnalysisResult = ( + value: unknown, + actionInput: unknown, + state: DataAnalysisState +): { + valid: boolean; + reasons: string[]; + validation_findings: AnalysisValidationFinding[]; + verified_values: unknown[]; +} => { + const result = recordValue(value, "result"); + const columns = recordValue(result, "columns"); + const rows = recordValue(result, "rows"); + const rowCount = recordValue(result, "row_count"); + const auditLogId = directString(result, "audit_log_id"); + const expectedColumns = recordStringArray(actionInput, "expected_columns"); + const missingColumns = expectedColumns.filter((column) => !Array.isArray(columns) || !columns.includes(column)); + const reasons = [ + ...(Array.isArray(columns) ? [] : ["RESULT_COLUMNS_REQUIRED"]), + ...(Array.isArray(rows) ? [] : ["RESULT_ROWS_REQUIRED"]), + ...(typeof rowCount === "number" && rowCount >= 0 ? [] : ["RESULT_ROW_COUNT_REQUIRED"]), + ...(auditLogId ? [] : ["RESULT_AUDIT_LOG_REQUIRED"]), + ...missingColumns.map((column) => `RESULT_EXPECTED_COLUMN_MISSING:${column}`) + ]; + const structuralFindings: AnalysisValidationFinding[] = reasons.map((reason) => ({ + code: reason, + message: `Result contract failed: ${reason}.`, + severity: "error" + })); + const attempt = state.queryAttempts?.find((candidate) => candidate.id === state.currentQueryAttemptId); + const verification = Array.isArray(columns) && Array.isArray(rows) && typeof rowCount === "number" + ? verifyAnalysisResult({ + columns: columns.filter((column): column is string => typeof column === "string"), + rows, + rowCount + }, attempt?.assertions ?? []) + : { valid: false, findings: [], verifiedValues: [] }; + const validationFindings = [...structuralFindings, ...verification.findings]; + return { + valid: validationFindings.every((finding) => finding.severity !== "error"), + reasons: validationFindings.map((finding) => finding.code), + validation_findings: validationFindings, + verified_values: verification.verifiedValues + }; +}; + +const nestedString = (value: unknown, parent: string, key: string): string | undefined => + directString(recordValue(value, parent), key); + +const nestedStringArray = (value: unknown, parent: string, key: string): string[] => + recordStringArray(recordValue(value, parent), key); + +const directString = (value: unknown, key: string): string | undefined => { + const field = recordValue(value, key); + return typeof field === "string" && field.length > 0 ? field : undefined; +}; + +const recordStringArray = (value: unknown, key: string): string[] => { + const field = recordValue(value, key); + return Array.isArray(field) + ? field.filter((item): item is string => typeof item === "string" && item.length > 0) + : []; +}; + +const recordArray = (value: unknown, key: string): unknown[] => { + const field = recordValue(value, key); + return Array.isArray(field) ? field : []; +}; + +const recordValue = (value: unknown, key: string): unknown => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record)[key] + : undefined; + +export type RunProtocolContextProjection = ContextPackageRef | ActionContextProjection; diff --git a/packages/agent-runtime/src/protocol/sql-semantic-validator.test.ts b/packages/agent-runtime/src/protocol/sql-semantic-validator.test.ts new file mode 100644 index 0000000..e347684 --- /dev/null +++ b/packages/agent-runtime/src/protocol/sql-semantic-validator.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; + +import { createAnalysisAssertions } from "./analysis-contract.js"; +import { validateSqlSemantics } from "./sql-semantic-validator.js"; + +describe("SQL semantic validator", () => { + it("accepts SQL that satisfies source, aggregate, grain and inclusive end-date constraints", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "按大区计算完整验证期订单数", + sourceTables: ["orders"], + sqlConstraints: [ + { kind: "aggregate", function: "count", alias: "order_count" }, + { kind: "group_by", columns: ["region"] }, + { + kind: "time_range", + column: "order_date", + start: "2023-07-01", + end: "2023-12-31", + endInclusive: true + } + ] + }]); + const sql = ` + SELECT region, COUNT(*) AS order_count + FROM orders + WHERE order_date >= '2023-07-01' AND order_date < '2024-01-01' + GROUP BY region + `; + + expect(validateSqlSemantics(sql, "sqlite", assertions)).toEqual([]); + }); + + it("accepts COUNT star when the grounded contract requires the star operand", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "计算订单总数", + sourceTables: ["orders"], + sqlConstraints: [{ kind: "aggregate", function: "COUNT", column: "*", alias: "order_count" }] + }]); + + const findings = validateSqlSemantics( + "SELECT COUNT(*) AS order_count FROM orders", + "sqlite", + assertions + ); + + expect(findings).toEqual([]); + }); + + it("rejects a query against the wrong source table", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "统计订单", + sourceTables: ["orders"] + }]); + + const findings = validateSqlSemantics("SELECT COUNT(*) FROM customers", "sqlite", assertions); + + expect(findings[0]?.code).toBe("SQL_SEMANTIC_SOURCE_MISSING:orders"); + }); + + it("rejects an aggregate with the wrong function or alias", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "统计订单数", + sqlConstraints: [{ kind: "aggregate", function: "count", alias: "order_count" }] + }]); + + const findings = validateSqlSemantics("SELECT SUM(amount) AS order_count FROM orders", "sqlite", assertions); + + expect(findings[0]?.code).toBe("SQL_SEMANTIC_AGGREGATE_MISSING:count"); + expect(findings[0]?.message).toContain("Expected COUNT AS order_count"); + expect(findings[0]?.message).toContain("observed SUM(amount) AS order_count"); + }); + + it("reports the exact expected and observed aliases for aggregate mismatches", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "metric", + description: "统计订单数", + sqlConstraints: [{ kind: "aggregate", function: "COUNT", column: "*", alias: "total_orders" }] + }]); + + const findings = validateSqlSemantics( + "SELECT COUNT(*) AS order_count FROM orders", + "sqlite", + assertions + ); + + expect(findings[0]?.message).toContain("Expected COUNT(*) AS total_orders"); + expect(findings[0]?.message).toContain("observed COUNT(*) AS order_count"); + }); + + it("rejects a missing group-by grain", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "grain", + description: "按大区统计", + sqlConstraints: [{ kind: "group_by", columns: ["region"] }] + }]); + + const findings = validateSqlSemantics("SELECT COUNT(*) FROM orders", "sqlite", assertions); + + expect(findings[0]?.code).toBe("SQL_SEMANTIC_GROUP_BY_MISSING:region"); + }); + + it("rejects an inclusive end date that omits the final day", () => { + const assertions = createAnalysisAssertions("R1", [{ + kind: "filter", + description: "包含完整验证期", + sqlConstraints: [{ + kind: "time_range", + column: "order_date", + start: "2023-07-01", + end: "2023-12-31", + endInclusive: true + }] + }]); + const sql = "SELECT COUNT(*) FROM orders WHERE order_date >= '2023-07-01' AND order_date < '2023-12-31'"; + + const findings = validateSqlSemantics(sql, "sqlite", assertions); + + expect(findings.map((finding) => finding.code)).toContain("SQL_SEMANTIC_TIME_END_MISSING:order_date"); + }); + + it("returns a stable finding for unparseable SQL", () => { + const findings = validateSqlSemantics("SELECT FROM", "sqlite", []); + + expect(findings[0]?.code).toBe("SQL_SEMANTIC_PARSE_FAILED"); + }); +}); diff --git a/packages/agent-runtime/src/protocol/sql-semantic-validator.ts b/packages/agent-runtime/src/protocol/sql-semantic-validator.ts new file mode 100644 index 0000000..7e51fef --- /dev/null +++ b/packages/agent-runtime/src/protocol/sql-semantic-validator.ts @@ -0,0 +1,247 @@ +import sqlParser from "node-sql-parser"; + +import type { + AnalysisAssertion, + AnalysisScalar, + AnalysisValidationFinding, + SqlSemanticConstraint +} from "./analysis-contract.js"; + +type SqlParseResult = { + tableList: string[]; + columnList: string[]; + ast: unknown; +}; + +type SqlPredicate = { + column: string; + operator: string; + value: AnalysisScalar; +}; + +type SqlAggregate = { + name: string; + column?: string; + alias?: string; +}; + +const parser = new sqlParser.Parser(); + +/** Compare parsed SQL semantics with the selected authoritative analysis assertions. */ +export const validateSqlSemantics = ( + sql: string, + dialect: string | undefined, + assertions: AnalysisAssertion[] +): AnalysisValidationFinding[] => { + let parsed: SqlParseResult; + try { + parsed = parser.parse(sql, { database: parserDialect(dialect) }) as SqlParseResult; + } catch (error) { + return [finding( + "SQL_SEMANTIC_PARSE_FAILED", + `SQL semantic parsing failed: ${error instanceof Error ? error.message : String(error)}` + )]; + } + if (Array.isArray(parsed.ast)) { + return [finding("SQL_SEMANTIC_MULTIPLE_STATEMENTS", "Exactly one SELECT statement is required.")]; + } + const ast = asRecord(parsed.ast); + if (ast.type !== "select") { + return [finding("SQL_SEMANTIC_SELECT_REQUIRED", "SQL semantic validation requires a SELECT statement.")]; + } + const tables = parsed.tableList.map((entry) => normalizeIdentifier(entry.split("::").at(-1) ?? entry)); + const columns = parsed.columnList.map((entry) => normalizeIdentifier(entry.split("::").at(-1) ?? entry)); + const aggregates = collectAggregates(ast.columns); + const groupBy = collectGroupBy(ast.groupby); + const predicates = collectPredicates(ast.where); + return assertions.flatMap((assertion) => { + if (assertion.kind === "manual") return []; + const constraints: SqlSemanticConstraint[] = [ + ...assertion.sourceTables.map((table) => ({ kind: "source" as const, table })), + ...assertion.sqlConstraints + ]; + return constraints.flatMap((constraint) => validateConstraint( + constraint, + { aggregates, columns, groupBy, predicates, tables }, + assertion.id + )); + }); +}; + +const validateConstraint = ( + constraint: SqlSemanticConstraint, + sql: { + aggregates: SqlAggregate[]; + columns: string[]; + groupBy: string[]; + predicates: SqlPredicate[]; + tables: string[]; + }, + assertionId: string +): AnalysisValidationFinding[] => { + if (constraint.kind === "source") { + return sql.tables.includes(normalizeIdentifier(constraint.table)) + ? [] + : [finding(`SQL_SEMANTIC_SOURCE_MISSING:${constraint.table}`, `Required source ${constraint.table} is missing.`, assertionId)]; + } + if (constraint.kind === "column") { + return sql.columns.includes(normalizeIdentifier(constraint.column)) + ? [] + : [finding(`SQL_SEMANTIC_COLUMN_MISSING:${constraint.column}`, `Required column ${constraint.column} is missing.`, assertionId)]; + } + if (constraint.kind === "aggregate") { + const matches = sql.aggregates.some((aggregate) => + aggregate.name === constraint.function.toLowerCase() + && (!constraint.column || aggregate.column === normalizeIdentifier(constraint.column)) + && (!constraint.alias || aggregate.alias === normalizeIdentifier(constraint.alias))); + return matches ? [] : [finding( + `SQL_SEMANTIC_AGGREGATE_MISSING:${constraint.function}`, + `Expected ${formatAggregate({ + name: constraint.function, + ...(constraint.column ? { column: normalizeIdentifier(constraint.column) } : {}), + ...(constraint.alias ? { alias: normalizeIdentifier(constraint.alias) } : {}) + })}, but observed ${formatObservedAggregates(sql.aggregates)}.`, + assertionId + )]; + } + if (constraint.kind === "group_by") { + const missing = constraint.columns.filter((column) => !sql.groupBy.includes(normalizeIdentifier(column))); + return missing.map((column) => finding( + `SQL_SEMANTIC_GROUP_BY_MISSING:${column}`, + `Required group-by column ${column} is missing.`, + assertionId + )); + } + if (constraint.kind === "filter") { + return predicateExists(sql.predicates, constraint.column, operatorToken(constraint.operator), constraint.value) + ? [] + : [finding( + `SQL_SEMANTIC_FILTER_MISSING:${constraint.column}:${constraint.operator}`, + `Required filter ${constraint.column} ${constraint.operator} is missing.`, + assertionId + )]; + } + const end = constraint.endInclusive ? nextDate(constraint.end) : constraint.end; + const hasStart = predicateExists(sql.predicates, constraint.column, ">=", constraint.start); + const hasEnd = predicateExists(sql.predicates, constraint.column, "<", end); + return [ + ...(hasStart ? [] : [finding( + `SQL_SEMANTIC_TIME_START_MISSING:${constraint.column}`, + `Required start boundary ${constraint.start} is missing.`, + assertionId + )]), + ...(hasEnd ? [] : [finding( + `SQL_SEMANTIC_TIME_END_MISSING:${constraint.column}`, + `Required half-open end boundary ${end} is missing.`, + assertionId + )]) + ]; +}; + +const collectAggregates = (value: unknown): SqlAggregate[] => { + if (!Array.isArray(value)) return []; + return value.flatMap((item) => { + const record = asRecord(item); + const expression = asRecord(record.expr); + if (expression.type !== "aggr_func" || typeof expression.name !== "string") return []; + const argument = asRecord(asRecord(expression.args).expr); + const column = argument.type === "column_ref" && typeof argument.column === "string" + ? normalizeIdentifier(argument.column) + : argument.type === "star" && argument.value === "*" + ? "*" + : undefined; + return [{ + name: expression.name.toLowerCase(), + ...(column ? { column } : {}), + ...(typeof record.as === "string" ? { alias: normalizeIdentifier(record.as) } : {}) + }]; + }); +}; + +const formatAggregate = (aggregate: SqlAggregate): string => { + const argument = aggregate.column === undefined ? "" : `(${aggregate.column})`; + const alias = aggregate.alias === undefined ? "" : ` AS ${aggregate.alias}`; + return `${aggregate.name.toUpperCase()}${argument}${alias}`; +}; + +const formatObservedAggregates = (aggregates: SqlAggregate[]): string => + aggregates.length === 0 ? "no aggregate expressions" : aggregates.map(formatAggregate).join(", "); + +const collectGroupBy = (value: unknown): string[] => { + const columns = asRecord(value).columns; + if (!Array.isArray(columns)) return []; + return columns.flatMap((column) => { + const record = asRecord(column); + return record.type === "column_ref" && typeof record.column === "string" + ? [normalizeIdentifier(record.column)] + : []; + }); +}; + +const collectPredicates = (value: unknown): SqlPredicate[] => { + const record = asRecord(value); + if (record.type !== "binary_expr" || typeof record.operator !== "string") return []; + if (record.operator.toUpperCase() === "AND") { + return [...collectPredicates(record.left), ...collectPredicates(record.right)]; + } + const left = asRecord(record.left); + const right = asRecord(record.right); + if (left.type !== "column_ref" || typeof left.column !== "string") return []; + const scalar = sqlLiteral(right); + return scalar === undefined ? [] : [{ + column: normalizeIdentifier(left.column), + operator: record.operator, + value: scalar + }]; +}; + +const sqlLiteral = (value: Record): AnalysisScalar | undefined => { + if (["single_quote_string", "double_quote_string", "string"].includes(String(value.type))) { + return typeof value.value === "string" ? value.value : undefined; + } + if (value.type === "number") return typeof value.value === "number" ? value.value : Number(value.value); + if (value.type === "bool") return Boolean(value.value); + if (value.type === "null") return null; + return undefined; +}; + +const predicateExists = ( + predicates: SqlPredicate[], + column: string, + operator: string, + value: AnalysisScalar +): boolean => predicates.some((predicate) => predicate.column === normalizeIdentifier(column) + && predicate.operator === operator && predicate.value === value); + +const operatorToken = (operator: "eq" | "gt" | "gte" | "lt" | "lte"): string => ({ + eq: "=", + gt: ">", + gte: ">=", + lt: "<", + lte: "<=" +})[operator]; + +const nextDate = (value: string): string => { + const matched = /^(\d{4})-(\d{2})-(\d{2})$/u.exec(value); + if (!matched) return value; + const date = new Date(Date.UTC(Number(matched[1]), Number(matched[2]) - 1, Number(matched[3]))); + date.setUTCDate(date.getUTCDate() + 1); + return date.toISOString().slice(0, 10); +}; + +const parserDialect = (dialect: string | undefined): string => { + const normalized = dialect?.toLowerCase(); + if (normalized === "postgres" || normalized === "postgresql") return "Postgresql"; + if (normalized === "mysql") return "MySQL"; + return "SQLite"; +}; + +const normalizeIdentifier = (value: string): string => value.replace(/^[`"[]|[`"\]]$/gu, "").toLowerCase(); +const asRecord = (value: unknown): Record => + typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +const finding = (code: string, message: string, assertionId?: string): AnalysisValidationFinding => ({ + code, + message, + severity: "error", + ...(assertionId ? { assertionId } : {}) +}); diff --git a/packages/agent-runtime/src/protocol/types.ts b/packages/agent-runtime/src/protocol/types.ts new file mode 100644 index 0000000..5397bde --- /dev/null +++ b/packages/agent-runtime/src/protocol/types.ts @@ -0,0 +1,115 @@ +export type ContextPackageRef = { + packageId: string; + revision: number; + eventId?: string; +}; + +export type ProtocolCompletionDecision = + | { status: "completed"; evaluatedContextPackageRef: ContextPackageRef; evidenceRefs: string[] } + | { + status: "degraded"; + evaluatedContextPackageRef: ContextPackageRef; + reasons: string[]; + evidenceRefs: string[]; + } + | { + status: "partial"; + evaluatedContextPackageRef: ContextPackageRef; + missing: string[]; + evidenceRefs: string[]; + } + | { status: "continue"; reasons: string[]; allowedActions: string[] } + | { status: "failed"; reasons: string[] }; + +export type ProtocolGuardResult = + | { allowed: true } + | { allowed: false; reasonCode: string; message?: string }; + +export type ProtocolGuard = (input: { + actionInput: unknown; + actionName: string; + state: TState; +}) => ProtocolGuardResult; + +export type ProtocolTransition = { + targetPhase: string; + when(input: { actionName: string; state: TState }): boolean; +}; + +export type ProtocolPhaseDefinition = { + allowedActions: string[]; + actionGuards?: Record[]>; + transitions: ProtocolTransition[]; +}; + +export type AgentProtocolDefinition = { + id: string; + version: string; + initialPhase: string; + phases: Record>; + createInitialState(input: { contextPackageRef: ContextPackageRef; runId: string }): TState; + completionPolicy(input: { + contextPackageRef: ContextPackageRef; + state: TState; + }): ProtocolCompletionDecision; +}; + +export type ProtocolActionRecord = { + actionId: string; + actionName: string; + status: "requested" | "rejected" | "succeeded" | "failed"; + inputContextPackageRef: ContextPackageRef; + outputContextPackageRef?: ContextPackageRef; + reasonCode?: string; +}; + +export type ProtocolRunState = { + protocolId: string; + protocolVersion: string; + runId: string; + segmentId: string; + revision: number; + phase: string; + status: "active" | "waiting" | "terminal" | "handed_off"; + contextPackageRef: ContextPackageRef; + actions: ProtocolActionRecord[]; + completionRejections: number; + domain: TDomainState; + terminalDecision?: ProtocolCompletionDecision; +}; + +export type ProtocolEvent = { + eventId: string; + type: string; + runId: string; + segmentId: string; + protocolId: string; + protocolVersion: string; + revision: number; + payload?: unknown; +}; + +export interface ProtocolStateStore { + create( + state: ProtocolRunState, + events?: ProtocolEvent[] + ): ProtocolRunState; + find(runId: string, segmentId?: string): ProtocolRunState | undefined; + get(runId: string, segmentId?: string): ProtocolRunState; + compareAndSet( + state: ProtocolRunState, + expectedRevision: number, + events?: ProtocolEvent[] + ): ProtocolRunState; + transitionSegment(input: { + current: ProtocolRunState; + expectedRevision: number; + next: ProtocolRunState; + events?: ProtocolEvent[]; + }): { + current: ProtocolRunState; + next: ProtocolRunState; + }; + acknowledgeEvent(event: ProtocolEvent): void; + pendingEvents(runId: string): ProtocolEvent[]; +} diff --git a/packages/agent-runtime/src/runtime-limits.test.ts b/packages/agent-runtime/src/runtime-limits.test.ts new file mode 100644 index 0000000..2fb6eeb --- /dev/null +++ b/packages/agent-runtime/src/runtime-limits.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import { AGENT_MAX_STEPS, SQL_MAX_EXECUTION_COUNT } from "./runtime-limits.js"; +import { + AGENT_RUNTIME_LIMIT_DEFINITIONS, + AGENT_RUNTIME_LIMITS +} from "./config/agent-runtime-limits.js"; + +describe("data analysis runtime limits", () => { + it("reserves enough headroom for complex analyses to execute and commit their final claims", () => { + expect(AGENT_MAX_STEPS).toBeGreaterThanOrEqual(80); + expect(SQL_MAX_EXECUTION_COUNT).toBeGreaterThanOrEqual(60); + }); + + it("keeps every runtime and context limit in one documented configuration registry", () => { + expect(Object.keys(AGENT_RUNTIME_LIMIT_DEFINITIONS)).toEqual(Object.keys(AGENT_RUNTIME_LIMITS)); + for (const definition of Object.values(AGENT_RUNTIME_LIMIT_DEFINITIONS)) { + expect(definition.env).toMatch(/^DATAFOUNDRY_[A-Z0-9_]+$/u); + expect(definition.description.length).toBeGreaterThan(20); + expect(definition.defaultValue).toBeGreaterThanOrEqual(definition.min); + expect(definition.defaultValue).toBeLessThanOrEqual(definition.max); + } + }); + + it("centralizes the contract grounding attempt budget", () => { + expect(AGENT_RUNTIME_LIMIT_DEFINITIONS.contractGrounderMaxAttempts).toMatchObject({ + defaultValue: 2, + env: "DATAFOUNDRY_CONTRACT_GROUNDER_MAX_ATTEMPTS" + }); + expect(AGENT_RUNTIME_LIMITS.contractGrounderMaxAttempts).toBe(2); + }); +}); diff --git a/packages/agent-runtime/src/runtime-limits.ts b/packages/agent-runtime/src/runtime-limits.ts index 7f936b1..5f91ace 100644 --- a/packages/agent-runtime/src/runtime-limits.ts +++ b/packages/agent-runtime/src/runtime-limits.ts @@ -1,9 +1,6 @@ -// Agent run execution limits. Context shaping limits live in context/inventory/context-limits.ts. +import { AGENT_RUNTIME_LIMITS } from "./config/agent-runtime-limits.js"; -// ReAct agents routinely run dozens of steps; 6 was far too tight for iterative analysis. -export const AGENT_MAX_STEPS = 50; +// Backward-compatible named exports. Definitions and environment overrides live in the central registry. +export const AGENT_MAX_STEPS = AGENT_RUNTIME_LIMITS.agentMaxSteps; -// Per-datasource SQL execution budget. Counted per schema capability (one datasource), -// so multi-datasource analysis is not starved by a single source's iteration. -// Total SQL across a run is still bounded by AGENT_MAX_STEPS. -export const SQL_MAX_EXECUTION_COUNT = 20; +export const SQL_MAX_EXECUTION_COUNT = AGENT_RUNTIME_LIMITS.sqlMaxExecutionCount; diff --git a/packages/agent-runtime/src/semantic/datalink-semantic-provider.test.ts b/packages/agent-runtime/src/semantic/datalink-semantic-provider.test.ts new file mode 100644 index 0000000..57a7d0a --- /dev/null +++ b/packages/agent-runtime/src/semantic/datalink-semantic-provider.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; + +import { DataLinkSemanticProvider } from "./datalink-semantic-provider.js"; +import { SemanticProviderError } from "./types.js"; + +const request = { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1", + query: "monthly revenue" +}; + +describe("DataLinkSemanticProvider", () => { + it("uses DataLink explore and preserves inference metadata", async () => { + const calls: unknown[] = []; + const provider = new DataLinkSemanticProvider({ + callTool: async (name, args) => { + calls.push({ name, args }); + return { + snapshot_id: "graph-7", + nodes: [{ id: "concept:revenue", source: "llm_inference", confidence: 0.72 }] + }; + } + }); + + const result = await provider.resolve(request); + + expect(calls).toEqual([{ + name: "datalink_explore", + args: { query: "monthly revenue", mask_credential: true } + }]); + expect(result).toMatchObject({ + snapshotId: "graph-7", + trust: "inferred", + capabilities: ["graph-explore"] + }); + }); + + it("marks authorization errors as non-fallback", async () => { + const provider = new DataLinkSemanticProvider({ + callTool: async () => { + throw new Error("HTTP 403 forbidden"); + } + }); + + await expect(provider.resolve(request)).rejects.toMatchObject({ + code: "DATALINK_NOT_AUTHORIZED", + fallbackAllowed: false + } satisfies Partial); + }); + + it("treats policy MCP error text as provider failure", async () => { + const provider = new DataLinkSemanticProvider({ + callTool: async () => "Error executing tool datalink_explore: MCP_TIMEOUT" + }); + + await expect(provider.resolve(request)).rejects.toMatchObject({ + code: "DATALINK_UNAVAILABLE", + fallbackAllowed: true + } satisfies Partial); + }); + + it.each([ + "", + " ", + "No results found for \"orders\". Try different keywords.", + { nodes: [] }, + { content: "No results found for orders" } + ])("treats empty semantic output as a fallback-allowed failure", async (value) => { + const provider = new DataLinkSemanticProvider({ callTool: async () => value }); + + await expect(provider.resolve(request)).rejects.toMatchObject({ + code: "DATALINK_EMPTY_RESULT", + fallbackAllowed: true + } satisfies Partial); + }); +}); diff --git a/packages/agent-runtime/src/semantic/datalink-semantic-provider.ts b/packages/agent-runtime/src/semantic/datalink-semantic-provider.ts new file mode 100644 index 0000000..8d92f85 --- /dev/null +++ b/packages/agent-runtime/src/semantic/datalink-semantic-provider.ts @@ -0,0 +1,97 @@ +import { + SemanticProviderError, + type SemanticProvider, + type SemanticProviderResult, + type SemanticRequest, + type SemanticTrust +} from "./types.js"; + +export type DataLinkToolClient = { + callTool(name: string, args: Record): Promise; +}; + +/** Resolve semantic context through the DataLink MCP explore capability. */ +export class DataLinkSemanticProvider implements SemanticProvider { + readonly id = "datalink" as const; + + constructor(private readonly client: DataLinkToolClient) {} + + async resolve(request: SemanticRequest): Promise { + try { + const value = await this.client.callTool("datalink_explore", { + query: request.query, + mask_credential: true + }); + if (typeof value === "string" && /^Error executing tool\b/iu.test(value)) { + throw new Error(value); + } + if (isEmptySemanticResult(value)) { + throw new SemanticProviderError("DATALINK_EMPTY_RESULT", true); + } + return { + value, + capabilities: ["graph-explore"], + trust: inferTrust(value), + warnings: [], + ...optionalSnapshotId(value) + }; + } catch (error) { + throw mapDataLinkError(error); + } + } +} + +const inferTrust = (value: unknown): SemanticTrust => { + if (typeof value === "string" && value.trim().length > 0) return "inferred"; + const nodes = recordValue(value, "nodes"); + if (!Array.isArray(nodes) || nodes.length === 0) return "unknown"; + const sources = nodes.map((node) => recordString(node, "source")); + if (sources.every((source) => source === "authoritative")) return "authoritative"; + if (sources.every((source) => source === "verified" || source === "authoritative")) return "verified"; + return "inferred"; +}; + +const optionalSnapshotId = (value: unknown): { snapshotId?: string } => { + const snapshotId = recordString(value, "snapshot_id"); + return snapshotId ? { snapshotId } : {}; +}; + +const mapDataLinkError = (error: unknown): SemanticProviderError => { + if (error instanceof SemanticProviderError) { + return error; + } + const message = error instanceof Error ? error.message : String(error); + if (/\b(?:401|403)\b|unauthori[sz]ed|forbidden/i.test(message)) { + return new SemanticProviderError("DATALINK_NOT_AUTHORIZED", false, { cause: error }); + } + if (/\b(?:400|404|409|422)\b|policy|invalid request/i.test(message)) { + return new SemanticProviderError("DATALINK_REQUEST_REJECTED", false, { cause: error }); + } + return new SemanticProviderError("DATALINK_UNAVAILABLE", true, { cause: error }); +}; + +const isEmptySemanticResult = (value: unknown): boolean => { + if (typeof value === "string") { + const text = value.trim(); + return text.length === 0 || /^No results found\b/iu.test(text); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return value === undefined || value === null; + } + const nodes = recordValue(value, "nodes"); + if (Array.isArray(nodes)) { + return nodes.length === 0; + } + const content = recordValue(value, "content"); + return typeof content === "string" && isEmptySemanticResult(content); +}; + +const recordValue = (value: unknown, key: string): unknown => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record)[key] + : undefined; + +const recordString = (value: unknown, key: string): string | undefined => { + const field = recordValue(value, key); + return typeof field === "string" && field.length > 0 ? field : undefined; +}; diff --git a/packages/agent-runtime/src/semantic/default-semantic-provider.test.ts b/packages/agent-runtime/src/semantic/default-semantic-provider.test.ts new file mode 100644 index 0000000..253a6e4 --- /dev/null +++ b/packages/agent-runtime/src/semantic/default-semantic-provider.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { createDefaultSemanticProvider } from "./default-semantic-provider.js"; + +const request = { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "3", + query: "revenue" +}; + +describe("createDefaultSemanticProvider", () => { + it("uses the selected DataLink MCP capability by default", async () => { + const calls: unknown[] = []; + const provider = createDefaultSemanticProvider({ + tools: { + datalink_explore: { + execute: async (input: unknown) => { + calls.push(input); + return { snapshot_id: "snapshot-1", nodes: [{ id: "metric:revenue", source: "verified" }] }; + } + } + } + }); + + await expect(provider.resolve(request)).resolves.toMatchObject({ + provider: "datalink", + mode: "live", + snapshotId: "snapshot-1" + }); + expect(calls).toEqual([{ query: "revenue", mask_credential: true }]); + }); +}); diff --git a/packages/agent-runtime/src/semantic/default-semantic-provider.ts b/packages/agent-runtime/src/semantic/default-semantic-provider.ts new file mode 100644 index 0000000..47bc7b2 --- /dev/null +++ b/packages/agent-runtime/src/semantic/default-semantic-provider.ts @@ -0,0 +1,26 @@ +import { DataLinkSemanticProvider } from "./datalink-semantic-provider.js"; +import { LocalSemanticProvider } from "./local-semantic-provider.js"; +import { SemanticProviderChain } from "./semantic-provider-chain.js"; + +type SemanticTool = { + execute?: (...args: unknown[]) => unknown | Promise; +}; + +/** Build the production semantic chain from the run's policy-selected capabilities. */ +export const createDefaultSemanticProvider = (input: { + tools: Record; +}): SemanticProviderChain => { + const live = new DataLinkSemanticProvider({ + callTool: async (name, args) => { + const execute = input.tools[name]?.execute; + if (!execute) { + throw new Error(`MCP_TOOL_UNAVAILABLE:${name}`); + } + return execute(args); + } + }); + const local = new LocalSemanticProvider({ + inspectSchema: async (request) => request.physicalSchema + }); + return new SemanticProviderChain({ live, local }); +}; diff --git a/packages/agent-runtime/src/semantic/local-semantic-provider.test.ts b/packages/agent-runtime/src/semantic/local-semantic-provider.test.ts new file mode 100644 index 0000000..f943836 --- /dev/null +++ b/packages/agent-runtime/src/semantic/local-semantic-provider.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; + +import { LocalSemanticProvider } from "./local-semantic-provider.js"; +import { SemanticProviderError } from "./types.js"; + +const request = { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "orders-db", + datasourceRevision: "schema-v1", + query: "monthly revenue" +}; + +describe("LocalSemanticProvider", () => { + it("returns only deterministic physical schema context", async () => { + const provider = new LocalSemanticProvider({ + inspectSchema: async () => ({ + tables: [{ name: "orders", columns: [{ name: "amount", type: "decimal" }] }], + metrics: [{ name: "revenue", expression: "sum(amount)" }], + inferred_joins: [{ left: "orders.customer_id", right: "customers.id" }] + }) + }); + + await expect(provider.resolve(request)).resolves.toEqual({ + value: { + tables: [{ name: "orders", columns: [{ name: "amount", type: "decimal" }] }] + }, + capabilities: ["physical-schema"], + trust: "verified", + warnings: ["LOCAL_SEMANTIC_LIMITED_TO_PHYSICAL_SCHEMA"] + }); + }); + + it("reports unavailable schema as a fallback-safe error", async () => { + const provider = new LocalSemanticProvider({ inspectSchema: async () => undefined }); + + await expect(provider.resolve(request)).rejects.toMatchObject({ + code: "LOCAL_SEMANTIC_UNAVAILABLE", + fallbackAllowed: true + } satisfies Partial); + }); + + it("uses schema already inspected by the governed data action", async () => { + const provider = new LocalSemanticProvider({ + inspectSchema: async () => { + throw new Error("SHOULD_NOT_REINSPECT"); + } + }); + + await expect(provider.resolve({ + ...request, + physicalSchema: { tables: [{ name: "orders", columns: [] }] } + })).resolves.toMatchObject({ + value: { tables: [{ name: "orders", columns: [] }] }, + capabilities: ["physical-schema"] + }); + }); +}); diff --git a/packages/agent-runtime/src/semantic/local-semantic-provider.ts b/packages/agent-runtime/src/semantic/local-semantic-provider.ts new file mode 100644 index 0000000..a12b7cd --- /dev/null +++ b/packages/agent-runtime/src/semantic/local-semantic-provider.ts @@ -0,0 +1,36 @@ +import { + SemanticProviderError, + type SemanticProvider, + type SemanticProviderResult, + type SemanticRequest +} from "./types.js"; + +export type LocalSchemaInspector = { + inspectSchema(request: SemanticRequest): Promise; +}; + +/** Provide a deterministic schema-only semantic fallback without inferred business meaning. */ +export class LocalSemanticProvider implements SemanticProvider { + readonly id = "local" as const; + + constructor(private readonly inspector: LocalSchemaInspector) {} + + async resolve(request: SemanticRequest): Promise { + const schema = request.physicalSchema ?? await this.inspector.inspectSchema(request); + const tables = recordValue(schema, "tables"); + if (!Array.isArray(tables)) { + throw new SemanticProviderError("LOCAL_SEMANTIC_UNAVAILABLE", true); + } + return { + value: { tables }, + capabilities: ["physical-schema"], + trust: "verified", + warnings: ["LOCAL_SEMANTIC_LIMITED_TO_PHYSICAL_SCHEMA"] + }; + } +} + +const recordValue = (value: unknown, key: string): unknown => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record)[key] + : undefined; diff --git a/packages/agent-runtime/src/semantic/semantic-provider-chain.test.ts b/packages/agent-runtime/src/semantic/semantic-provider-chain.test.ts new file mode 100644 index 0000000..69c4ac6 --- /dev/null +++ b/packages/agent-runtime/src/semantic/semantic-provider-chain.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; + +import { SemanticProviderChain } from "./semantic-provider-chain.js"; +import { SemanticProviderError, type SemanticProvider } from "./types.js"; + +const request = { + userId: "user-1", + workspaceId: "workspace-1", + datasourceId: "datasource-1", + datasourceRevision: "schema-v1", + query: "monthly revenue" +}; + +describe("SemanticProviderChain", () => { + it("uses live DataLink as the default provider", async () => { + const chain = new SemanticProviderChain({ + live: provider("datalink", { nodes: ["revenue"] }), + local: provider("local", { tables: [] }) + }); + + const result = await chain.resolve(request); + + expect(result).toMatchObject({ provider: "datalink", mode: "live", value: { nodes: ["revenue"] } }); + }); + + it("uses a fresh DataLink snapshot after a transient live failure", async () => { + const chain = new SemanticProviderChain({ + live: failingProvider("DATALINK_TIMEOUT", true), + local: provider("local", { tables: [] }), + now: () => 1000 + }); + chain.cacheSnapshot(request, { + value: { nodes: ["cached-revenue"] }, + snapshotId: "snapshot-1", + expiresAt: 2000 + }); + + const result = await chain.resolve(request); + + expect(result).toMatchObject({ + provider: "datalink", + mode: "cached", + snapshotId: "snapshot-1", + value: { nodes: ["cached-revenue"] } + }); + }); + + it("uses the deterministic local provider when no fresh snapshot exists", async () => { + const chain = new SemanticProviderChain({ + live: failingProvider("DATALINK_UNAVAILABLE", true), + local: provider("local", { tables: ["orders"] }), + now: () => 3000 + }); + chain.cacheSnapshot(request, { value: {}, snapshotId: "expired", expiresAt: 2000 }); + + const result = await chain.resolve(request); + + expect(result).toMatchObject({ provider: "local", mode: "fallback", value: { tables: ["orders"] } }); + expect(result.fallbackReason).toBe("DATALINK_UNAVAILABLE"); + }); + + it("preserves an empty-result fallback reason", async () => { + const chain = new SemanticProviderChain({ + live: failingProvider("DATALINK_EMPTY_RESULT", true), + local: provider("local", { tables: ["orders"] }) + }); + + const result = await chain.resolve(request); + + expect(result).toMatchObject({ + provider: "local", + mode: "fallback", + fallbackReason: "DATALINK_EMPTY_RESULT" + }); + }); + + it("does not hide DataLink authorization failures with a fallback", async () => { + const chain = new SemanticProviderChain({ + live: failingProvider("DATALINK_NOT_AUTHORIZED", false), + local: provider("local", { tables: ["orders"] }) + }); + + await expect(chain.resolve(request)).rejects.toThrow("DATALINK_NOT_AUTHORIZED"); + }); +}); + +const provider = (id: "datalink" | "local", value: unknown): SemanticProvider => ({ + id, + resolve: async () => ({ + value, + capabilities: id === "datalink" ? ["graph-explore"] : ["physical-schema"], + trust: id === "datalink" ? "inferred" : "verified", + warnings: [] + }) +}); + +const failingProvider = (code: string, fallbackAllowed: boolean): SemanticProvider => ({ + id: "datalink", + resolve: async () => { + throw new SemanticProviderError(code, fallbackAllowed); + } +}); diff --git a/packages/agent-runtime/src/semantic/semantic-provider-chain.ts b/packages/agent-runtime/src/semantic/semantic-provider-chain.ts new file mode 100644 index 0000000..5ec93ae --- /dev/null +++ b/packages/agent-runtime/src/semantic/semantic-provider-chain.ts @@ -0,0 +1,93 @@ +import { + SemanticProviderError, + type SemanticProvider, + type SemanticProviderResult, + type SemanticRequest, + type SemanticResolution +} from "./types.js"; + +export type SemanticSnapshot = { + value: unknown; + snapshotId: string; + expiresAt: number; + capabilities?: string[]; + trust?: SemanticProviderResult["trust"]; + warnings?: string[]; +}; + +export type SemanticProviderChainOptions = { + live: SemanticProvider; + local: SemanticProvider; + now?: () => number; +}; + +export class SemanticProviderChain { + private readonly snapshots = new Map(); + + constructor(private readonly options: SemanticProviderChainOptions) {} + + cacheSnapshot(request: SemanticRequest, snapshot: SemanticSnapshot): void { + this.snapshots.set(snapshotKey(request), snapshot); + } + + async resolve(request: SemanticRequest): Promise { + let fallbackReason: string | undefined; + try { + const result = await this.options.live.resolve(request); + return { + ...result, + provider: "datalink", + mode: "live", + datasourceRevision: request.datasourceRevision + }; + } catch (error) { + if (!(error instanceof SemanticProviderError) || !error.fallbackAllowed) { + throw error; + } + fallbackReason = error.code; + } + const snapshot = this.snapshots.get(snapshotKey(request)); + if (snapshot && snapshot.expiresAt > (this.options.now?.() ?? Date.now())) { + return { + value: snapshot.value, + capabilities: snapshot.capabilities ?? ["graph-explore"], + trust: snapshot.trust ?? "inferred", + warnings: snapshot.warnings ?? [], + snapshotId: snapshot.snapshotId, + provider: "datalink", + mode: "cached", + datasourceRevision: request.datasourceRevision, + ...(fallbackReason ? { fallbackReason } : {}) + }; + } + try { + const result = await this.options.local.resolve(request); + return { + ...result, + provider: "local", + mode: "fallback", + datasourceRevision: request.datasourceRevision, + ...(fallbackReason ? { fallbackReason } : {}) + }; + } catch (error) { + const reason = error instanceof SemanticProviderError ? error.code : "LOCAL_SEMANTIC_UNAVAILABLE"; + return { + value: undefined, + capabilities: [], + trust: "unknown", + warnings: [reason], + provider: "none", + mode: "unavailable", + datasourceRevision: request.datasourceRevision, + ...(fallbackReason ? { fallbackReason } : {}) + }; + } + } +} + +const snapshotKey = (request: SemanticRequest): string => [ + request.userId, + request.workspaceId, + request.datasourceId, + request.datasourceRevision +].join(":"); diff --git a/packages/agent-runtime/src/semantic/types.ts b/packages/agent-runtime/src/semantic/types.ts new file mode 100644 index 0000000..1f1bc88 --- /dev/null +++ b/packages/agent-runtime/src/semantic/types.ts @@ -0,0 +1,41 @@ +export type SemanticTrust = "authoritative" | "verified" | "inferred" | "unknown"; + +export type SemanticRequest = { + userId: string; + workspaceId: string; + datasourceId: string; + datasourceRevision: string; + query: string; + physicalSchema?: unknown; +}; + +export type SemanticProviderResult = { + value: unknown; + capabilities: string[]; + trust: SemanticTrust; + warnings: string[]; + snapshotId?: string; +}; + +export interface SemanticProvider { + id: "datalink" | "local"; + resolve(request: SemanticRequest): Promise; +} + +export type SemanticResolution = SemanticProviderResult & { + provider: "datalink" | "local" | "none"; + mode: "live" | "cached" | "fallback" | "unavailable"; + datasourceRevision: string; + fallbackReason?: string; +}; + +export class SemanticProviderError extends Error { + constructor( + readonly code: string, + readonly fallbackAllowed: boolean, + options?: ErrorOptions + ) { + super(code, options); + this.name = "SemanticProviderError"; + } +} diff --git a/packages/agent-runtime/src/testing.ts b/packages/agent-runtime/src/testing.ts index 62d200d..790cbd0 100644 --- a/packages/agent-runtime/src/testing.ts +++ b/packages/agent-runtime/src/testing.ts @@ -8,6 +8,36 @@ export { } from "./index.js"; export { createDataFoundryToolRegistry } from "./tools/data-tools.js"; export { GovernedToolFactory } from "./tools/governed-tool-factory.js"; +export { ActionRouter } from "./capabilities/action-router.js"; +export { CapabilityRegistry } from "./capabilities/capability-registry.js"; +export { createToolCapabilityPlugin } from "./capabilities/tool-capability-plugin.js"; +export type * from "./capabilities/types.js"; +export { ToolExecutionError, toToolExecutionError, toolErrorObservation } from "./errors/tool-execution-error.js"; +export type * from "./errors/tool-execution-error.js"; +export { validateProtocolDefinition } from "./protocol/definition-validator.js"; +export { evaluateProtocolHandoff } from "./protocol/protocol-handoff.js"; +export { ProtocolHandoffCoordinator } from "./protocol/protocol-handoff-coordinator.js"; +export { InMemoryProtocolStateStore } from "./protocol/in-memory-protocol-state-store.js"; +export { ProtocolRegistry } from "./protocol/protocol-registry.js"; +export { ProtocolRouter } from "./protocol/protocol-router.js"; +export { ProtocolRuntime } from "./protocol/protocol-runtime.js"; +export { + createModelProtocolClassifier, + createProtocolClassificationPrompt +} from "./protocol/model-protocol-classifier.js"; +export { createRunProtocolBoundary } from "./protocol/run-protocol-boundary.js"; +export { createGeneralTaskProtocol } from "./protocol/protocols/general-task.js"; +export { createDataAnalysisProtocol } from "./protocol/protocols/data-analysis.js"; +export type * from "./protocol/protocol-handoff.js"; +export type * from "./protocol/protocol-handoff-coordinator.js"; +export type * from "./protocol/protocol-router.js"; +export type * from "./protocol/protocol-runtime.js"; +export type * from "./protocol/types.js"; +export { DataLinkSemanticProvider } from "./semantic/datalink-semantic-provider.js"; +export { LocalSemanticProvider } from "./semantic/local-semantic-provider.js"; +export { SemanticProviderChain } from "./semantic/semantic-provider-chain.js"; +export { createDefaultSemanticProvider } from "./semantic/default-semantic-provider.js"; +export type * from "./semantic/types.js"; export { createContextItem, hashContextContent } from "./context/inventory/context-item.js"; export type { diff --git a/packages/agent-runtime/src/tools/data-tools-cache.test.ts b/packages/agent-runtime/src/tools/data-tools-cache.test.ts new file mode 100644 index 0000000..84d85e6 --- /dev/null +++ b/packages/agent-runtime/src/tools/data-tools-cache.test.ts @@ -0,0 +1,49 @@ +import type { DataGateway } from "@datafoundry/data-gateway"; +import { describe, expect, it } from "vitest"; + +import { createDataFoundryToolRegistry } from "./data-tools.js"; + +describe("data tool SQL reuse", () => { + it("reuses an exact successful SQL result within one run and schema", async () => { + let executionCount = 0; + const dataGateway = { + inspectSchema: async () => ({ datasource_id: "orders", dialect: "sqlite", tables: [] }), + runSqlReadonly: async () => { + executionCount += 1; + return { + columns: ["value"], + rows: [[1]], + row_count: 1, + audit_log_id: "audit-1", + artifact_id: "artifact-1", + elapsed_ms: 1 + }; + } + } as unknown as DataGateway; + const registry = createDataFoundryToolRegistry({ + dataGateway, + emitter: { emit: () => undefined }, + runContext: { + user_id: "user-1", + workspace_id: "workspace-1", + session_id: "session-1", + run_id: "run-cache", + user_input: "analyze orders", + chat_mode: "copilotkit", + enabled_datasource_ids: ["orders"], + selected_datasource_id: "orders", + model_name: "test-model" + } + }); + const schema = await registry.inspectSchema({ datasource_id: "orders" }); + + const first = await registry.runSqlReadonly({ schema_id: schema.schema_id, sql: "SELECT 1", limit: 10 }); + const second = await registry.runSqlReadonly({ schema_id: schema.schema_id, sql: "SELECT 1", limit: 10 }); + await registry.runSqlReadonly({ schema_id: schema.schema_id, sql: "SELECT 1", limit: 20 }); + + expect(first.cache_hit).toBeUndefined(); + expect(second).toMatchObject({ cache_hit: true, result: { audit_log_id: "audit-1" } }); + expect(executionCount).toBe(2); + expect(registry.state.sql_execution_count).toBe(2); + }); +}); diff --git a/packages/agent-runtime/src/tools/data-tools.ts b/packages/agent-runtime/src/tools/data-tools.ts index dc3e48d..67855a4 100644 --- a/packages/agent-runtime/src/tools/data-tools.ts +++ b/packages/agent-runtime/src/tools/data-tools.ts @@ -11,6 +11,7 @@ import { createActivitySnapshot, createArtifactEvent, createCustomEvent } from " import { SQL_MAX_EXECUTION_COUNT } from "../runtime-limits.js"; import { createTokenUsageCorrelationStore } from "../stream/token-usage-correlation.js"; import type { AgentRunContext, AgUiEventEmitter } from "../types.js"; +import { enrichSqlDialectError, validateSqlDialect } from "./sql-dialect-validation.js"; type DataToolExecutionOptions = { toolCallId?: string; @@ -36,6 +37,7 @@ type TokenUsageCorrelationStore = ReturnType() }; const resultMetadata = new WeakMap(); + const sqlResultCache = new Map(); const listDataSources = async (toolInput: { enabled_only?: boolean } = {}): Promise => { throwIfAborted(input.abortSignal); @@ -156,7 +163,11 @@ export const createDataFoundryToolRegistry = (input: CreateDataFoundryToolRegist ...(input.abortSignal ? { signal: input.abortSignal } : {}) }); const schema_id = `schema_${randomUUID()}`; - state.schema_capabilities.set(schema_id, { datasource_id: datasourceId, schema_id }); + state.schema_capabilities.set(schema_id, { + datasource_id: datasourceId, + ...(result.dialect ? { dialect: result.dialect } : {}), + schema_id + }); const rawResult = { ...result, schema_id }; resultMetadata.set(rawResult, { datasourceId, stepId }); return rawResult; @@ -170,6 +181,9 @@ export const createDataFoundryToolRegistry = (input: CreateDataFoundryToolRegist toolInput: { schema_id: string; sql: string; + assertion_ids?: string[]; + requirement_ids?: string[]; + expected_columns?: string[]; limit?: number; timeout_ms?: number; }, @@ -181,6 +195,18 @@ export const createDataFoundryToolRegistry = (input: CreateDataFoundryToolRegist throw new Error("SCHEMA_REQUIRED_BEFORE_SQL"); } const datasourceId = capability.datasource_id; + const dialectIssues = validateSqlDialect(toolInput.sql, capability.dialect); + if (dialectIssues.length > 0) { + const issue = dialectIssues[0]; + throw new Error(`SQL_DIALECT_UNSUPPORTED:${issue?.dialect}:${issue?.code}:${issue?.hint}`); + } + const cacheKey = sqlCacheKey(toolInput); + const cached = sqlResultCache.get(cacheKey); + if (cached) { + const rawResult = { ...cached, cache_hit: true as const }; + resultMetadata.set(rawResult, { datasourceId, stepId: `sql-cache-${randomUUID()}` }); + return rawResult; + } state.sql_execution_count += 1; const datasourceCount = (state.sql_execution_count_by_datasource.get(datasourceId) ?? 0) + 1; @@ -227,11 +253,12 @@ export const createDataFoundryToolRegistry = (input: CreateDataFoundryToolRegist } emitSqlReferences(input, datasourceId, result); const rawResult = { result, sql: toolInput.sql }; + sqlResultCache.set(cacheKey, rawResult); resultMetadata.set(rawResult, { datasourceId, stepId }); return rawResult; } catch (error) { emitFailedStep(input, stepId, "run_sql_readonly", "Run read-only SQL", error); - throw error; + throw enrichSqlDialectError(error, capability.dialect); } }; @@ -309,6 +336,18 @@ export const createDataFoundryToolRegistry = (input: CreateDataFoundryToolRegist }; }; +const sqlCacheKey = (input: { + schema_id: string; + sql: string; + limit?: number; + timeout_ms?: number; +}): string => [ + input.schema_id, + input.sql.trim().replace(/;\s*$/u, ""), + input.limit ?? "default", + input.timeout_ms ?? "default" +].join("\u0000"); + type DataToolExecutors = Pick; const createMastraDataTools = (executors: DataToolExecutors): ToolRegistry["mastraTools"] => ({ @@ -357,6 +396,9 @@ const createMastraDataTools = (executors: DataToolExecutors): ToolRegistry["mast inputSchema: z.object({ schema_id: z.string(), sql: z.string(), + assertion_ids: z.array(z.string().min(1)).max(32).optional(), + requirement_ids: z.array(z.string().min(1)).max(16).optional(), + expected_columns: z.array(z.string().min(1)).max(100).optional(), limit: z.number().int().positive().max(1000).optional(), timeout_ms: z.number().int().positive().max(30000).optional() }), @@ -365,6 +407,9 @@ const createMastraDataTools = (executors: DataToolExecutors): ToolRegistry["mast { schema_id: toolInput.schema_id, sql: toolInput.sql, + ...(toolInput.assertion_ids ? { assertion_ids: toolInput.assertion_ids } : {}), + ...(toolInput.requirement_ids ? { requirement_ids: toolInput.requirement_ids } : {}), + ...(toolInput.expected_columns ? { expected_columns: toolInput.expected_columns } : {}), ...(toolInput.limit ? { limit: toolInput.limit } : {}), ...(toolInput.timeout_ms ? { timeout_ms: toolInput.timeout_ms } : {}), }, diff --git a/packages/agent-runtime/src/tools/governed-tool-factory.test.ts b/packages/agent-runtime/src/tools/governed-tool-factory.test.ts new file mode 100644 index 0000000..d724e62 --- /dev/null +++ b/packages/agent-runtime/src/tools/governed-tool-factory.test.ts @@ -0,0 +1,203 @@ +import { EventType } from "@ag-ui/core"; +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { ActionRouter } from "../capabilities/action-router.js"; +import { CapabilityRegistry } from "../capabilities/capability-registry.js"; +import { toToolExecutionError } from "../errors/tool-execution-error.js"; +import { InMemoryProtocolStateStore } from "../protocol/in-memory-protocol-state-store.js"; +import { ProtocolRuntime } from "../protocol/protocol-runtime.js"; +import type { AgentProtocolDefinition } from "../protocol/types.js"; +import { createToolObservationBoundary } from "../context/tool-observation/tool-observation-boundary.js"; +import { ToolObservationDispatcher } from "../context/tool-observation/tool-observation-dispatcher.js"; +import { GovernedToolFactory } from "./governed-tool-factory.js"; + +describe("GovernedToolFactory protocol boundary", () => { + it("rejects a disallowed tool before its executor runs", async () => { + let executed = false; + const emitted: Array<{ content?: string; type?: string }> = []; + const runScope = { modelName: "test", resourceId: "user-1", runId: "run-1", sessionId: "session-1" }; + const boundary = createToolObservationBoundary({ + additionalAdapters: [{ + toolName: "test_tool", + resultType: "test", + sourceType: "tool-observation", + toContextItems: () => [] + }], + identity: runScope + }); + const dispatcher = new ToolObservationDispatcher(boundary.packager, runScope); + const registry = new CapabilityRegistry(); + registry.register({ + manifest: { id: "test-tools", version: "1", provides: ["test_tool"] }, + actions: [{ + name: "test_tool", + exposure: "agent", + inputSchema: z.unknown(), + outputSchema: z.unknown(), + idempotency: "none", + execute: async () => { + executed = true; + return { ok: true }; + } + }] + }); + const runtime = new ProtocolRuntime(createProtocol(), new InMemoryProtocolStateStore()); + runtime.start({ + runId: "run-1", + segmentId: "segment-1", + contextPackageRef: { packageId: "context-1", revision: 0 } + }); + const actionRouter = new ActionRouter(registry, runtime, { + serverPolicy: () => ({ allowed: true }), + projectContext: () => ({ packageId: "context-1", revision: 1 }) + }); + const factory = new GovernedToolFactory(dispatcher, undefined, undefined, { + actionRouter, + emitter: { emit: (event) => emitted.push(event) }, + runId: "run-1", + segmentId: "segment-1" + }); + const rawTool: { execute(...args: unknown[]): Promise } = { + execute: async () => ({ ok: true }) + }; + const tool = factory.governTool("test_tool", rawTool); + + await expect(tool.execute?.({}, { agent: { toolCallId: "call-rejected" } })) + .rejects.toThrow("ACTION_NOT_ALLOWED_IN_PHASE:active:test_tool"); + expect(executed).toBe(false); + expect(emitted).toHaveLength(1); + expect(emitted[0]?.type).toBe(EventType.TOOL_CALL_RESULT); + expect(JSON.parse(emitted[0]?.content ?? "{}")).toEqual({ + ok: false, + isError: true, + error: { + code: "ACTION_NOT_ALLOWED_IN_PHASE", + category: "protocol", + message: "Tool test_tool is not allowed in protocol phase active.", + executionStatus: "not_started", + retryable: false + }, + recovery: { + strategy: "refresh_and_replan", + instruction: "Choose an action allowed in the current phase before calling this tool again.", + avoid: ["Do not repeat test_tool while the protocol remains in phase active."] + } + }); + }); + + it("preserves governed result callbacks after Action Router execution", async () => { + const runScope = { modelName: "test", resourceId: "user-1", runId: "run-1", sessionId: "session-1" }; + const boundary = createToolObservationBoundary({ + additionalAdapters: [{ + toolName: "test_tool", + resultType: "test", + sourceType: "tool-observation", + toContextItems: () => [] + }], + identity: runScope + }); + const dispatcher = new ToolObservationDispatcher(boundary.packager, runScope); + const callbacks: unknown[] = []; + const contextPackage = { + version: 2 as const, + packageId: "context-1", + revision: 1, + items: [], + groups: [], + sourceSnapshots: [], + artifactRefs: [], + auditRefs: [], + truncation: [] + }; + const factory = new GovernedToolFactory(dispatcher, (result) => { + callbacks.push(result); + }, undefined, { + actionRouter: { + execute: async () => ({ + rawResult: { ok: true }, + observation: { summary: "done" }, + contextPackageRef: { packageId: "context-1", revision: 1 }, + contextPackage + }) + }, + runId: "run-1", + segmentId: "segment-1" + }); + const tool = factory.governTool("test_tool", { + execute: async (..._args: unknown[]) => ({ shouldNotRun: true }) + }); + + const result = await tool.execute?.({}, { agent: { toolCallId: "call-1" } }); + + expect(result).toEqual({ summary: "done" }); + expect(callbacks).toHaveLength(1); + expect(callbacks[0]).toMatchObject({ + contextPackage, + rawResult: { ok: true }, + toolName: "test_tool", + toolCallId: "call-1" + }); + }); + + it("warns the model not to repeat an external action whose state commit failed", async () => { + const emitted: unknown[] = []; + const runScope = { modelName: "test", resourceId: "user-1", runId: "run-1", sessionId: "session-1" }; + const boundary = createToolObservationBoundary({ + additionalAdapters: [{ + toolName: "side_effect_tool", + resultType: "test", + sourceType: "tool-observation", + toContextItems: () => [] + }], + identity: runScope + }); + const dispatcher = new ToolObservationDispatcher(boundary.packager, runScope); + const factory = new GovernedToolFactory(dispatcher, undefined, undefined, { + actionRouter: { + execute: async () => { + throw toToolExecutionError(new Error("PROTOCOL_COMMIT_CONTENTION:complete_action"), { + executionStatus: "succeeded_uncommitted", + idempotency: "none", + toolName: "side_effect_tool" + }, { rawResult: { created: true } }); + } + }, + emitter: { emit: (event) => emitted.push(event) }, + runId: "run-1", + segmentId: "segment-1" + }); + const rawTool: { execute(...args: unknown[]): Promise } = { + execute: async () => ({ shouldNotRun: true }) + }; + const tool = factory.governTool("side_effect_tool", rawTool); + + await expect(tool.execute?.({}, { agent: { toolCallId: "call-contention" } })) + .rejects.toThrow("PROTOCOL_COMMIT_CONTENTION"); + + const content = (emitted[0] as { content?: string } | undefined)?.content; + expect(JSON.parse(content ?? "{}")).toMatchObject({ + ok: false, + isError: true, + error: { + code: "PROTOCOL_COMMIT_CONTENTION", + category: "concurrency", + executionStatus: "succeeded_uncommitted", + retryable: false + }, + recovery: { + strategy: "refresh_and_replan", + avoid: ["Do not immediately repeat side_effect_tool; its external execution already completed."] + } + }); + }); +}); + +const createProtocol = (): AgentProtocolDefinition> => ({ + id: "test/protocol", + version: "1", + initialPhase: "active", + phases: { active: { allowedActions: [], transitions: [] } }, + createInitialState: () => ({}), + completionPolicy: () => ({ status: "continue", reasons: ["not done"], allowedActions: [] }) +}); diff --git a/packages/agent-runtime/src/tools/governed-tool-factory.ts b/packages/agent-runtime/src/tools/governed-tool-factory.ts index 424e8ed..7f5ae1f 100644 --- a/packages/agent-runtime/src/tools/governed-tool-factory.ts +++ b/packages/agent-runtime/src/tools/governed-tool-factory.ts @@ -2,12 +2,15 @@ import type { ContextPackage } from "../context/inventory/context-package.js"; import type { ToolObservationDispatcher } from "../context/tool-observation/tool-observation-dispatcher.js"; import { toolObservationModelFromPackage } from "../context/tool-observation/tool-observation-projection-items.js"; import { createToolCallResult } from "../events.js"; +import { ToolExecutionError, toolErrorObservation } from "../errors/tool-execution-error.js"; import type { AgUiEventEmitter } from "../types.js"; +import type { ActionExecutionResult, ExecuteActionInput } from "../capabilities/action-router.js"; type ToolExecution = (...args: any[]) => unknown | Promise; type MastraToolExecuteOptions = { agent?: { toolCallId?: string }; + abortSignal?: AbortSignal; }; type ExecutableTool = { @@ -31,6 +34,7 @@ export type GovernedToolErrorHandler = (input: { }) => void | Promise; export type GovernedToolFactoryOptions = { + actionRouter?: { execute(input: ExecuteActionInput): Promise }; /** When set, the boundary emits an authoritative TOOL_CALL_RESULT for every governed tool. */ emitter?: AgUiEventEmitter; /** @@ -39,11 +43,18 @@ export type GovernedToolFactoryOptions = { * clobbering the suspend/resume contract. */ externallyResolvedToolNames?: Set; + runId?: string; + segmentId?: string; + getSegmentId?(): string; }; export class GovernedToolFactory { private readonly emitter: AgUiEventEmitter | undefined; private readonly externallyResolvedToolNames: Set; + private readonly actionRouter: GovernedToolFactoryOptions["actionRouter"]; + private readonly runId: string | undefined; + private readonly segmentId: string | undefined; + private readonly getSegmentId: (() => string) | undefined; constructor( private readonly dispatcher: ToolObservationDispatcher, @@ -53,6 +64,10 @@ export class GovernedToolFactory { ) { this.emitter = options.emitter; this.externallyResolvedToolNames = options.externallyResolvedToolNames ?? new Set(); + this.actionRouter = options.actionRouter; + this.runId = options.runId; + this.segmentId = options.segmentId; + this.getSegmentId = options.getSegmentId; } /** Wrap every registered tool at its execution boundary. */ @@ -76,6 +91,35 @@ export class GovernedToolFactory { const toolCallId = toolCallIdFromOptions(options); const toolInput = args[0]; try { + if (this.actionRouter) { + const segmentId = this.getSegmentId?.() ?? this.segmentId; + if (!this.runId || !segmentId) { + throw new Error("GOVERNED_ACTION_ROUTER_SCOPE_REQUIRED"); + } + const actionResult = await this.actionRouter.execute({ + actionId: toolCallId ?? `${toolName}:${Date.now()}`, + actionName: toolName, + runId: this.runId, + segmentId, + input: toolInput, + ...(options?.abortSignal ? { abortSignal: options.abortSignal } : {}), + invocationArgs: args.slice(1) + }); + if (this.onResult) { + if (!actionResult.contextPackage) { + throw new Error(`GOVERNED_CONTEXT_PACKAGE_REQUIRED:${toolName}`); + } + await this.onResult({ + contextPackage: actionResult.contextPackage as ContextPackage, + rawResult: actionResult.rawResult, + toolName, + ...(toolCallId ? { toolCallId } : {}), + ...(toolInput !== undefined ? { toolInput } : {}) + }); + } + this.emitToolCallResult(toolCallId, toolName, serializeToolResultContent(actionResult.observation)); + return actionResult.observation; + } const rawResult = await execute(...args); if (rawResult === undefined) { return undefined; @@ -92,9 +136,10 @@ export class GovernedToolFactory { this.emitToolCallResult(toolCallId, toolName, serializeToolResultContent(observation)); return observation; } catch (error) { + const errorObservation = toolErrorObservation(error, { toolName }); await this.onError?.({ error, - rawResult: undefined, + rawResult: error instanceof ToolExecutionError ? error.rawResult : undefined, toolName, ...(toolCallId ? { toolCallId } : {}), ...(toolInput !== undefined ? { toolInput } : {}) @@ -102,10 +147,7 @@ export class GovernedToolFactory { this.emitToolCallResult( toolCallId, toolName, - JSON.stringify({ - error: stringifyToolError(error), - isError: true, - }), + JSON.stringify(errorObservation), ); throw error; } @@ -140,17 +182,3 @@ const toolCallIdFromOptions = (options?: MastraToolExecuteOptions): string | und const serializeToolResultContent = (observation: unknown): string => typeof observation === "string" ? observation : JSON.stringify(observation); - -const stringifyToolError = (error: unknown): string => { - if (error instanceof Error) { - return error.message; - } - if (typeof error === "string") { - return error; - } - try { - return JSON.stringify(error); - } catch { - return String(error); - } -}; diff --git a/packages/agent-runtime/src/tools/sql-dialect-validation.test.ts b/packages/agent-runtime/src/tools/sql-dialect-validation.test.ts new file mode 100644 index 0000000..a4e3955 --- /dev/null +++ b/packages/agent-runtime/src/tools/sql-dialect-validation.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { enrichSqlDialectError, validateSqlDialect } from "./sql-dialect-validation.js"; + +describe("SQL dialect validation", () => { + it("rejects percentile_cont within group for SQLite with a repair hint", () => { + expect(validateSqlDialect( + "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY profit) FROM orders", + "sqlite" + )).toEqual([{ + code: "SQLITE_PERCENTILE_CONT_UNSUPPORTED", + dialect: "sqlite", + hint: "Use ROW_NUMBER and COUNT to select or interpolate ordered percentile rows." + }]); + }); + + it("rejects ilike for SQLite", () => { + expect(validateSqlDialect("SELECT * FROM orders WHERE name ILIKE '%a%'", "sqlite")).toEqual([ + expect.objectContaining({ code: "SQLITE_ILIKE_UNSUPPORTED", dialect: "sqlite" }) + ]); + }); + + it("allows supported SQLite read queries", () => { + expect(validateSqlDialect("SELECT region, SUM(profit) FROM orders GROUP BY region", "sqlite")).toEqual([]); + }); + + it("enriches a known SQLite UNION LIMIT error", () => { + const error = enrichSqlDialectError( + new Error("LIMIT clause should come after UNION ALL not before"), + "sqlite" + ); + + expect(error.message).toContain("SQLITE_UNION_LIMIT_POSITION"); + expect(error.message).toContain("Wrap the limited UNION branch in a subquery"); + }); +}); diff --git a/packages/agent-runtime/src/tools/sql-dialect-validation.ts b/packages/agent-runtime/src/tools/sql-dialect-validation.ts new file mode 100644 index 0000000..b3ff1d4 --- /dev/null +++ b/packages/agent-runtime/src/tools/sql-dialect-validation.ts @@ -0,0 +1,63 @@ +export type SqlDialectIssue = { + code: string; + dialect: string; + hint: string; +}; + +/** Detect common SQL constructs unsupported by the inspected datasource dialect. */ +export const validateSqlDialect = (sql: string, dialect: string | undefined): SqlDialectIssue[] => { + if (dialect?.toLowerCase() !== "sqlite") { + return []; + } + const issues: SqlDialectIssue[] = []; + if (/\bPERCENTILE_CONT\s*\([^)]*\)\s*WITHIN\s+GROUP\b/iu.test(sql)) { + issues.push({ + code: "SQLITE_PERCENTILE_CONT_UNSUPPORTED", + dialect: "sqlite", + hint: "Use ROW_NUMBER and COUNT to select or interpolate ordered percentile rows." + }); + } + if (/\bILIKE\b/iu.test(sql)) { + issues.push({ + code: "SQLITE_ILIKE_UNSUPPORTED", + dialect: "sqlite", + hint: "Use LIKE with COLLATE NOCASE for case-insensitive matching." + }); + } + if (/\bDATE_TRUNC\s*\(/iu.test(sql)) { + issues.push({ + code: "SQLITE_DATE_TRUNC_UNSUPPORTED", + dialect: "sqlite", + hint: "Use strftime with the required date grain." + }); + } + if (/\bQUALIFY\b/iu.test(sql)) { + issues.push({ + code: "SQLITE_QUALIFY_UNSUPPORTED", + dialect: "sqlite", + hint: "Move the window predicate into an outer SELECT WHERE clause." + }); + } + return issues; +}; + +/** Add stable dialect context and repair guidance to known backend syntax failures. */ +export const enrichSqlDialectError = (error: unknown, dialect: string | undefined): Error => { + const message = error instanceof Error ? error.message : String(error); + if (dialect?.toLowerCase() !== "sqlite") { + return error instanceof Error ? error : new Error(message); + } + if (/LIMIT clause should come after UNION(?: ALL)? not before/iu.test(message)) { + return new Error( + "SQL_DIALECT_ERROR:sqlite:SQLITE_UNION_LIMIT_POSITION:" + + "Wrap the limited UNION branch in a subquery, or apply LIMIT after the complete UNION." + ); + } + if (/near ["']?WITHIN["']?: syntax error/iu.test(message)) { + return new Error( + "SQL_DIALECT_ERROR:sqlite:SQLITE_WITHIN_GROUP_UNSUPPORTED:" + + "Replace WITHIN GROUP ordered-set aggregates with SQLite window functions." + ); + } + return error instanceof Error ? error : new Error(message); +}; diff --git a/packages/data-gateway/src/index.ts b/packages/data-gateway/src/index.ts index a04a194..4585d27 100644 --- a/packages/data-gateway/src/index.ts +++ b/packages/data-gateway/src/index.ts @@ -57,6 +57,7 @@ import type { DataGatewayPolicy, DataSourceAdapter, DataSourceRuntimePolicy, + DataSourceType, InspectSchemaInput, ListDataSourcesInput, PreviewTableInput, @@ -102,6 +103,9 @@ const DEFAULT_DATA_GATEWAY_POLICY: DataGatewayPolicy = { timeoutMs: 10000 }; +const sqlDialectForDataSourceType = (type: DataSourceType): string => + type === "csv" || type === "xlsx" ? "duckdb" : type; + export class LocalDataGateway implements DataGateway { private readonly artifactService: LocalArtifactService; @@ -158,6 +162,7 @@ export class LocalDataGateway implements DataGateway { return { datasource_id: input.datasource_id, + dialect: sqlDialectForDataSourceType(dataSource.type), tables: tableNames.size > 0 ? schema.tables.filter((table) => tableNames.has(table.name)) : schema.tables }; } diff --git a/packages/data-gateway/src/types.ts b/packages/data-gateway/src/types.ts index 98af69c..f8abf72 100644 --- a/packages/data-gateway/src/types.ts +++ b/packages/data-gateway/src/types.ts @@ -109,6 +109,7 @@ export type RunSqlReadonlyInput = { export type SchemaSummary = { datasource_id: string; + dialect?: string; tables: Array<{ name: string; columns: Array<{ diff --git a/packages/metadata/src/conversation-message-repository.test.ts b/packages/metadata/src/conversation-message-repository.test.ts new file mode 100644 index 0000000..ce9197d --- /dev/null +++ b/packages/metadata/src/conversation-message-repository.test.ts @@ -0,0 +1,50 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createMetadataStore } from "./index.js"; + +describe("ConversationMessageRepository", () => { + it("finds only the latest persisted assistant message for the requested run", () => { + const root = mkdtempSync(join(tmpdir(), "conversation-message-")); + try { + const metadata = createMetadataStore({ database_path: join(root, "metadata.sqlite") }); + metadata.sessions.create({ user_id: "dev-user", id: "session-1", title: "Conversation" }); + for (const runId of ["run-1", "run-2"]) { + metadata.runs.create({ + user_id: "dev-user", + id: runId, + session_id: "session-1", + user_input: "test", + status: "running" + }); + } + const append = (runId: string, role: "assistant" | "user", messageId: string): void => { + metadata.conversationMessages.append({ + id: `${runId}:${messageId}`, + user_id: "dev-user", + session_id: "session-1", + run_id: runId, + role, + source: role === "assistant" ? "agent" : "client", + message_id: messageId, + content_text: messageId + }); + }; + append("run-1", "assistant", "assistant-1"); + append("run-2", "assistant", "assistant-other-run"); + append("run-1", "user", "user-latest"); + append("run-1", "assistant", "assistant-2"); + + expect(metadata.conversationMessages.findLatestAssistantByRun({ + user_id: "dev-user", + session_id: "session-1", + run_id: "run-1" + })?.message_id).toBe("assistant-2"); + metadata.close(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index adc9330..66e8a02 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -1,4 +1,4 @@ -import type { BaseEvent, EventType } from "@ag-ui/core"; +import { EventType, type BaseEvent } from "@ag-ui/core"; import type { ArtifactSummary, ArtifactType, RunEventEnvelope } from "@datafoundry/contracts"; import { createHash } from "node:crypto"; import { mkdirSync } from "node:fs"; @@ -242,7 +242,23 @@ export type ContextPackageSnapshotRecord = { created_at: string; }; -export type CheckpointKind = "context-compiled" | "run-terminal" | "tool-result"; +export type ProtocolStateSnapshotRecord = { + user_id: string; + run_id: string; + segment_id: string; + protocol_id: string; + protocol_version: string; + revision: number; + phase: string; + status: string; + context_package_id: string; + context_package_revision: number; + state_json: string; + created_at: string; + updated_at: string; +}; + +export type CheckpointKind = "context-compiled" | "protocol-phase" | "run-terminal" | "tool-result"; export type CheckpointStatus = "stable" | "failed" | "terminal"; @@ -476,6 +492,21 @@ export type CreateContextPackageSnapshotInput = { plan?: unknown; }; +export type CompareAndSetProtocolStateInput = { + user_id: string; + run_id: string; + segment_id: string; + expected_revision: number; + state: unknown; +}; + +export type TransitionProtocolSegmentInput = { + user_id: string; + current: Omit; + next: Omit; + events?: unknown[]; +}; + export type CreateCheckpointInput = { id: string; user_id: string; @@ -622,6 +653,7 @@ export class MetadataStore { readonly interactions: InteractionRepository; readonly longTermMemories: LongTermMemoryRepository; readonly queryHistory: QueryHistoryRepository; + readonly protocolStates: ProtocolStateSnapshotRepository; readonly runEvents: RunEventRepository; readonly runs: RunRepository; readonly sessionBranches: SessionBranchRepository; @@ -661,6 +693,7 @@ export class MetadataStore { this.interactions = new InteractionRepository(db); this.longTermMemories = new LongTermMemoryRepository(db); this.queryHistory = new QueryHistoryRepository(db); + this.protocolStates = new ProtocolStateSnapshotRepository(db); this.secrets = new EncryptedSecretStore(db, secretMasterKey); this.sqlAuditLogs = new SqlAuditLogRepository(db); } @@ -1742,6 +1775,18 @@ export class RunEventRepository { constructor(private readonly db: DatabaseSync) {} append(input: WriteRunEventInput): RunEventRecord { + const protocolEventId = protocolEventIdFromBaseEvent(input.event); + if (protocolEventId) { + const existing = mapRunEventRow(this.db.prepare(` + SELECT * FROM run_events + WHERE user_id = ? AND run_id = ? + AND json_extract(payload_json, '$.value.eventId') = ? + LIMIT 1 + `).get(input.user_id, input.run_id, protocolEventId)); + if (existing) { + return existing; + } + } const seq = this.nextSeq({ user_id: input.user_id, run_id: input.run_id }); const createdAt = new Date().toISOString(); const id = `${input.run_id}:${seq}`; @@ -1801,6 +1846,15 @@ export class RunEventRepository { } } +const protocolEventIdFromBaseEvent = (event: BaseEvent): string | undefined => { + if (event.type !== EventType.CUSTOM || !isRecord(event) || !isRecord(event.value)) { + return undefined; + } + return typeof event.value.eventId === "string" && event.value.eventId.length > 0 + ? event.value.eventId + : undefined; +}; + export class ConversationMessageRepository { constructor(private readonly db: DatabaseSync) {} @@ -1940,6 +1994,26 @@ export class ConversationMessageRepository { return this.db.prepare(sql).all(...params).map(mapRequiredConversationMessageRow); } + findLatestAssistantByRun(input: { + user_id: string; + session_id: string; + run_id: string; + }): ConversationMessageRecord | undefined { + return mapConversationMessageRow( + this.db + .prepare( + ` + SELECT * FROM conversation_messages + WHERE user_id = ? AND session_id = ? AND run_id = ? + AND role = 'assistant' AND message_id IS NOT NULL + ORDER BY position DESC + LIMIT 1 + ` + ) + .get(input.user_id, input.session_id, input.run_id) + ); + } + private findByMessageId(input: { user_id: string; session_id: string; @@ -2302,6 +2376,199 @@ export class ContextPackageSnapshotRepository { } } +export class ProtocolStateSnapshotRepository { + constructor(private readonly db: DatabaseSync) {} + + compareAndSet(input: CompareAndSetProtocolStateInput): ProtocolStateSnapshotRecord { + const state = requireProtocolStateFields(input.state); + if (state.runId !== input.run_id || state.segmentId !== input.segment_id) { + throw new Error(`PROTOCOL_STATE_SCOPE_MISMATCH:${input.run_id}:${input.segment_id}`); + } + const current = this.find({ + user_id: input.user_id, + run_id: input.run_id, + segment_id: input.segment_id + }); + const currentRevision = current?.revision ?? -1; + if (currentRevision !== input.expected_revision) { + throw new Error( + `PROTOCOL_REVISION_CONFLICT:${input.run_id}:${input.segment_id}:` + + `${input.expected_revision}:${currentRevision}` + ); + } + if (state.revision !== input.expected_revision + 1) { + throw new Error( + `PROTOCOL_REVISION_INVALID:${input.run_id}:${input.segment_id}:${state.revision}` + ); + } + const contextExists = this.db.prepare(` + SELECT 1 AS found + FROM context_package_snapshots + WHERE user_id = ? AND package_id = ? AND revision = ? + `).get(input.user_id, state.contextPackageId, state.contextPackageRevision); + if (!contextExists) { + throw new Error( + `PROTOCOL_CONTEXT_REF_NOT_FOUND:${state.contextPackageId}@${state.contextPackageRevision}` + ); + } + const now = new Date().toISOString(); + if (!current) { + this.db.prepare(` + INSERT INTO protocol_state_snapshots ( + user_id, run_id, segment_id, protocol_id, protocol_version, revision, phase, status, + context_package_id, context_package_revision, state_json, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + input.user_id, + input.run_id, + input.segment_id, + state.protocolId, + state.protocolVersion, + state.revision, + state.phase, + state.status, + state.contextPackageId, + state.contextPackageRevision, + JSON.stringify(input.state), + now, + now + ); + } else { + const result = this.db.prepare(` + UPDATE protocol_state_snapshots + SET protocol_id = ?, protocol_version = ?, revision = ?, phase = ?, status = ?, + context_package_id = ?, context_package_revision = ?, state_json = ?, updated_at = ? + WHERE user_id = ? AND run_id = ? AND segment_id = ? AND revision = ? + `).run( + state.protocolId, + state.protocolVersion, + state.revision, + state.phase, + state.status, + state.contextPackageId, + state.contextPackageRevision, + JSON.stringify(input.state), + now, + input.user_id, + input.run_id, + input.segment_id, + input.expected_revision + ); + if (result.changes !== 1) { + throw new Error( + `PROTOCOL_REVISION_CONFLICT:${input.run_id}:${input.segment_id}:` + + `${input.expected_revision}:${currentRevision}` + ); + } + } + return this.get({ user_id: input.user_id, run_id: input.run_id, segment_id: input.segment_id }); + } + + compareAndSetWithEvents( + input: CompareAndSetProtocolStateInput, + events: unknown[] + ): ProtocolStateSnapshotRecord { + this.db.exec("BEGIN IMMEDIATE"); + try { + const state = this.compareAndSet(input); + this.appendEvents(input.user_id, events); + this.db.exec("COMMIT"); + return state; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + transitionSegment(input: TransitionProtocolSegmentInput): { + current: ProtocolStateSnapshotRecord; + next: ProtocolStateSnapshotRecord; + } { + this.db.exec("BEGIN IMMEDIATE"); + try { + const current = this.compareAndSet({ user_id: input.user_id, ...input.current }); + const next = this.compareAndSet({ user_id: input.user_id, ...input.next }); + this.appendEvents(input.user_id, input.events ?? []); + this.db.exec("COMMIT"); + return { current, next }; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + find(input: { + user_id: string; + run_id: string; + segment_id: string; + }): Optional { + return mapProtocolStateSnapshotRow(this.db.prepare(` + SELECT * FROM protocol_state_snapshots + WHERE user_id = ? AND run_id = ? AND segment_id = ? + `).get(input.user_id, input.run_id, input.segment_id)); + } + + get(input: { user_id: string; run_id: string; segment_id: string }): ProtocolStateSnapshotRecord { + const snapshot = this.find(input); + if (!snapshot) { + throw new Error(`Protocol state snapshot not found: ${input.run_id}:${input.segment_id}`); + } + return snapshot; + } + + latestByRun(input: { user_id: string; run_id: string }): Optional { + return mapProtocolStateSnapshotRow(this.db.prepare(` + SELECT * FROM protocol_state_snapshots + WHERE user_id = ? AND run_id = ? + ORDER BY rowid DESC + LIMIT 1 + `).get(input.user_id, input.run_id)); + } + + acknowledgeEvent(input: { user_id: string; event_id: string }): void { + this.db.prepare(` + UPDATE protocol_event_journal + SET published_at = ? + WHERE user_id = ? AND event_id = ? + `).run(new Date().toISOString(), input.user_id, input.event_id); + } + + pendingEvents(input: { user_id: string; run_id: string }): unknown[] { + return this.db.prepare(` + SELECT event_json FROM protocol_event_journal + WHERE user_id = ? AND run_id = ? AND published_at IS NULL + ORDER BY created_at ASC, rowid ASC + `).all(input.user_id, input.run_id).map((row) => { + if (!isRecord(row) || typeof row.event_json !== "string") { + throw new Error(`PROTOCOL_EVENT_JOURNAL_INVALID:${input.run_id}`); + } + return JSON.parse(row.event_json) as unknown; + }); + } + + private appendEvents(userId: string, events: unknown[]): void { + const statement = this.db.prepare(` + INSERT OR IGNORE INTO protocol_event_journal ( + user_id, run_id, segment_id, event_id, revision, event_type, event_json, created_at, published_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL) + `); + const now = new Date().toISOString(); + for (const event of events) { + const fields = requireProtocolEventFields(event); + statement.run( + userId, + fields.runId, + fields.segmentId, + fields.eventId, + fields.revision, + fields.type, + JSON.stringify(event), + now + ); + } + } +} + export class CheckpointRepository { constructor(private readonly db: DatabaseSync) {} @@ -3287,6 +3554,28 @@ const runMigrations = (db: DatabaseSync): void => { CREATE INDEX IF NOT EXISTS idx_context_package_snapshots_user_run ON context_package_snapshots(user_id, run_id, revision); + CREATE TABLE IF NOT EXISTS protocol_state_snapshots ( + user_id TEXT NOT NULL, + run_id TEXT NOT NULL, + segment_id TEXT NOT NULL, + protocol_id TEXT NOT NULL, + protocol_version TEXT NOT NULL, + revision INTEGER NOT NULL, + phase TEXT NOT NULL, + status TEXT NOT NULL, + context_package_id TEXT NOT NULL, + context_package_revision INTEGER NOT NULL, + state_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (user_id, run_id, segment_id), + FOREIGN KEY (user_id, run_id) REFERENCES runs(user_id, id), + FOREIGN KEY (user_id, context_package_id, context_package_revision) + REFERENCES context_package_snapshots(user_id, package_id, revision) + ); + CREATE INDEX IF NOT EXISTS idx_protocol_state_snapshots_user_run + ON protocol_state_snapshots(user_id, run_id, updated_at); + CREATE TABLE IF NOT EXISTS checkpoints ( id TEXT NOT NULL, user_id TEXT NOT NULL, @@ -3511,6 +3800,12 @@ const runMigrations = (db: DatabaseSync): void => { runSchemaMigration(db, "0015_trace_section_phase_key", "Ensure trace section phase key", () => { ensureTraceSectionPhaseKeyColumn(db); }); + runSchemaMigration(db, "0016_protocol_state_snapshots", "Ensure protocol state snapshot schema", () => { + initializeProtocolStateSnapshotSchema(db); + }); + runSchemaMigration(db, "0017_protocol_event_journal", "Ensure protocol event journal schema", () => { + initializeProtocolEventJournalSchema(db); + }); }; const initializeSchemaMigrationTable = (db: DatabaseSync): void => { @@ -3637,6 +3932,52 @@ const initializeTraceSectionSchema = (db: DatabaseSync): void => { `); }; +const initializeProtocolStateSnapshotSchema = (db: DatabaseSync): void => { + db.exec(` + CREATE TABLE IF NOT EXISTS protocol_state_snapshots ( + user_id TEXT NOT NULL, + run_id TEXT NOT NULL, + segment_id TEXT NOT NULL, + protocol_id TEXT NOT NULL, + protocol_version TEXT NOT NULL, + revision INTEGER NOT NULL, + phase TEXT NOT NULL, + status TEXT NOT NULL, + context_package_id TEXT NOT NULL, + context_package_revision INTEGER NOT NULL, + state_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (user_id, run_id, segment_id), + FOREIGN KEY (user_id, run_id) REFERENCES runs(user_id, id), + FOREIGN KEY (user_id, context_package_id, context_package_revision) + REFERENCES context_package_snapshots(user_id, package_id, revision) + ); + CREATE INDEX IF NOT EXISTS idx_protocol_state_snapshots_user_run + ON protocol_state_snapshots(user_id, run_id, updated_at); + `); +}; + +const initializeProtocolEventJournalSchema = (db: DatabaseSync): void => { + db.exec(` + CREATE TABLE IF NOT EXISTS protocol_event_journal ( + user_id TEXT NOT NULL, + run_id TEXT NOT NULL, + segment_id TEXT NOT NULL, + event_id TEXT NOT NULL, + revision INTEGER NOT NULL, + event_type TEXT NOT NULL, + event_json TEXT NOT NULL, + created_at TEXT NOT NULL, + published_at TEXT, + PRIMARY KEY (user_id, event_id), + FOREIGN KEY (user_id, run_id) REFERENCES runs(user_id, id) + ); + CREATE INDEX IF NOT EXISTS idx_protocol_event_journal_pending + ON protocol_event_journal(user_id, run_id, published_at, created_at); + `); +}; + const initializeAuthSchema = (db: DatabaseSync): void => { ensureColumn(db, "users", "email_verified_at", "TEXT"); ensureColumn(db, "users", "disabled_at", "TEXT"); @@ -4566,6 +4907,73 @@ const mapContextPackageSnapshotRow = (row: unknown): Optional => { + if (!isRecord(row)) { + return undefined; + } + return { + user_id: requiredString(row, "user_id"), + run_id: requiredString(row, "run_id"), + segment_id: requiredString(row, "segment_id"), + protocol_id: requiredString(row, "protocol_id"), + protocol_version: requiredString(row, "protocol_version"), + revision: requiredNumber(row, "revision"), + phase: requiredString(row, "phase"), + status: requiredString(row, "status"), + context_package_id: requiredString(row, "context_package_id"), + context_package_revision: requiredNumber(row, "context_package_revision"), + state_json: requiredString(row, "state_json"), + created_at: requiredString(row, "created_at"), + updated_at: requiredString(row, "updated_at") + }; +}; + +const requireProtocolStateFields = (value: unknown): { + protocolId: string; + protocolVersion: string; + runId: string; + segmentId: string; + revision: number; + phase: string; + status: string; + contextPackageId: string; + contextPackageRevision: number; +} => { + if (!isRecord(value) || !isRecord(value.contextPackageRef)) { + throw new Error("PROTOCOL_STATE_INVALID"); + } + return { + protocolId: requiredString(value, "protocolId"), + protocolVersion: requiredString(value, "protocolVersion"), + runId: requiredString(value, "runId"), + segmentId: requiredString(value, "segmentId"), + revision: requiredNumber(value, "revision"), + phase: requiredString(value, "phase"), + status: requiredString(value, "status"), + contextPackageId: requiredString(value.contextPackageRef, "packageId"), + contextPackageRevision: requiredNumber(value.contextPackageRef, "revision") + }; +}; + +const requireProtocolEventFields = (value: unknown): { + eventId: string; + runId: string; + segmentId: string; + revision: number; + type: string; +} => { + if (!isRecord(value)) { + throw new Error("PROTOCOL_EVENT_INVALID"); + } + return { + eventId: requiredString(value, "eventId"), + runId: requiredString(value, "runId"), + segmentId: requiredString(value, "segmentId"), + revision: requiredNumber(value, "revision"), + type: requiredString(value, "type") + }; +}; + const mapCheckpointRow = (row: unknown): Optional => { if (!isRecord(row)) { return undefined; diff --git a/packages/metadata/src/run-event-dedupe.test.ts b/packages/metadata/src/run-event-dedupe.test.ts new file mode 100644 index 0000000..e856e83 --- /dev/null +++ b/packages/metadata/src/run-event-dedupe.test.ts @@ -0,0 +1,39 @@ +import { EventType } from "@ag-ui/core"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { createMetadataStore, RunEventWriter } from "./index.js"; + +describe("RunEventWriter protocol event deduplication", () => { + it("does not append a replayed protocol journal event twice", () => { + const root = mkdtempSync(join(tmpdir(), "protocol-event-dedupe-")); + try { + const metadata = createMetadataStore({ database_path: join(root, "metadata.sqlite") }); + metadata.sessions.create({ user_id: "dev-user", id: "session-1", title: "Protocol" }); + metadata.runs.create({ + user_id: "dev-user", + id: "run-1", + session_id: "session-1", + user_input: "test", + status: "running" + }); + const writer = new RunEventWriter(metadata.runEvents); + const event = { + type: EventType.CUSTOM, + name: "protocol.state.updated", + value: { eventId: "protocol-event-1" } + }; + + const first = writer.write({ user_id: "dev-user", run_id: "run-1", session_id: "session-1", event }); + const replay = writer.write({ user_id: "dev-user", run_id: "run-1", session_id: "session-1", event }); + + expect(replay.seq).toBe(first.seq); + expect(writer.replay({ user_id: "dev-user", run_id: "run-1" })).toHaveLength(1); + metadata.close(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/smoke-agent-protocol-deepseek.mjs b/scripts/smoke-agent-protocol-deepseek.mjs new file mode 100644 index 0000000..32a5a94 --- /dev/null +++ b/scripts/smoke-agent-protocol-deepseek.mjs @@ -0,0 +1,215 @@ +import assert from "node:assert/strict"; +import "dotenv/config"; + +import { + createModelProtocolClassifier, + createRunProtocolBoundary +} from "../packages/agent-runtime/dist/testing.js"; +import { createModelProvider } from "../packages/providers/dist/index.js"; + +const provider = createModelProvider(process.env); +assert.notEqual(provider.kind, "mock", "DeepSeek credentials are required; fake LLM is not allowed"); +const classifier = createModelProtocolClassifier(provider); +const candidates = [ + { protocolId: "general-task", protocolVersion: "1" }, + { protocolId: "data-analysis", protocolVersion: "1" } +]; + +const general = await classifier({ + candidates, + value: { userText: "用两句话解释什么是数据仓库，不需要查询任何数据源" } +}); +assert.equal(general.protocolId, "general-task"); +assert.ok(general.confidence >= 0.75); + +const eventTypes = []; +const analyticBoundary = await createRunProtocolBoundary({ + runId: `deepseek-protocol-${Date.now()}`, + userInput: "比较订单在不同月份的趋势，找出下降最明显的月份", + authorizedProtocolIds: ["general-task", "data-analysis"], + initialContextPackageRef: { packageId: "deepseek-context", revision: 0 }, + tools: {}, + classifier, + projectContext: () => ({ packageId: "deepseek-context", revision: 0 }), + runtimeOptions: { onEvent: (event) => eventTypes.push(event.type) } +}); +assert.equal(analyticBoundary.route.definition.id, "data-analysis"); +assert.equal(analyticBoundary.route.source, "classifier"); +assert.deepEqual(eventTypes.slice(0, 5), [ + "protocol.route.requested", + "protocol.route.classified", + "protocol.route.resolved", + "protocol.run.started", + "protocol.phase.entered" +]); + +console.log( + `Real DeepSeek protocol smoke OK: general=${general.confidence.toFixed(2)}, ` + + `analytic=${analyticBoundary.route.definition.id}.` +); + +async function runApiScenarios(baseUrl) { + const stamp = Date.now(); + const generalEvents = await runAgent(baseUrl, { + threadId: `protocol-general-${stamp}`, + runId: `protocol-general-run-${stamp}`, + prompt: "用一句话解释什么是星型模型，不要查询数据源。", + runConfig: { + protocol: { id: "general-task", version: "1" }, + activeLlmProfileId: "server-default", + enabledDatasourceIds: [], + enabledKnowledgeIds: [], + enabledMcpServerIds: [], + enabledSkillIds: [] + } + }); + assertProtocolTerminal(generalEvents, "protocol.run.completed"); + assert.equal(protocolEvent(generalEvents, "protocol.route.resolved")?.value?.payload?.source, "explicit"); + + const dataEvents = await runAgent(baseUrl, { + threadId: `protocol-data-${stamp}`, + runId: `protocol-data-run-${stamp}`, + prompt: "查询 orders 表，计算订单总数和 gmv 总和。必须先检查 schema，再执行只读 SQL，并说明结果。", + runConfig: { + protocol: { id: "data-analysis", version: "1" }, + activeDatasourceId: "local-sqlite-orders", + activeLlmProfileId: "server-default", + enabledDatasourceIds: ["local-sqlite-orders"], + enabledKnowledgeIds: [], + enabledMcpServerIds: [], + enabledSkillIds: [] + } + }); + assertProtocolTerminal(dataEvents, "protocol.run.degraded"); + const degradedDecision = protocolEvent(dataEvents, "protocol.run.degraded")?.value?.payload?.decision; + assert.ok( + degradedDecision?.reasons?.includes("LOCAL_SEMANTIC_LIMITED_TO_PHYSICAL_SCHEMA"), + "Data E2E must disclose local semantic fallback when DataLink is disabled" + ); + const extractedRequirements = protocolEvent(dataEvents, "analysis.requirements.extracted") + ?.value?.payload?.requirements; + assert.ok(Array.isArray(extractedRequirements) && extractedRequirements.length > 0); + const groundingResult = dataEvents.find((event) => + event.type === "CUSTOM" + && event.name === "protocol.action.succeeded" + && event.value?.payload?.actionName === "analysis.contract.ground" + )?.value?.payload?.result; + assert.ok( + Array.isArray(groundingResult?.structuredRequirementIds) + && groundingResult.structuredRequirementIds.length === extractedRequirements.length, + `Every extracted requirement must be schema-grounded: ${JSON.stringify(groundingResult)}` + ); + assert.deepEqual(groundingResult.manualRequirementIds, []); + const actionOrder = dataEvents + .filter((event) => event.type === "CUSTOM" && event.name === "protocol.action.started") + .map((event) => event.value?.payload?.actionName); + assert.ok(actionOrder.indexOf("inspect_schema") >= 0); + assert.ok(actionOrder.indexOf("semantic.context.resolve") > actionOrder.indexOf("inspect_schema")); + assert.ok(actionOrder.indexOf("data.query.plan") > actionOrder.indexOf("semantic.context.resolve")); + assert.ok(actionOrder.indexOf("data.query.validate") > actionOrder.indexOf("data.query.plan")); + assert.ok(actionOrder.indexOf("run_sql_readonly") > actionOrder.indexOf("data.query.validate")); + assert.ok(actionOrder.includes("analysis.evidence.bind")); + const committedClaims = toolResults(dataEvents, "analysis_requirements_commit") + .flatMap((event) => parseClaims(event.content)); + assert.ok(committedClaims.length > 0, "Data E2E must commit requirement claims"); + assert.ok( + committedClaims.some((claim) => Array.isArray(claim.values) && claim.values.length > 0), + "Structured claims must carry runtime-verified values" + ); + + const handoffEvents = await runAgent(baseUrl, { + threadId: `protocol-handoff-${stamp}`, + runId: `protocol-handoff-run-${stamp}`, + prompt: "必须实际查询 orders 表的订单数。当前协议不适用时，先用 protocol_handoff 切换到 data-analysis@1。", + runConfig: { + protocol: { id: "general-task", version: "1" }, + activeDatasourceId: "local-sqlite-orders", + activeLlmProfileId: "server-default", + enabledDatasourceIds: ["local-sqlite-orders"], + enabledKnowledgeIds: [], + enabledMcpServerIds: [], + enabledSkillIds: [] + } + }); + assert.ok(protocolEvent(handoffEvents, "protocol.handoff.accepted")); + assertProtocolTerminal(handoffEvents, "protocol.run.degraded"); + console.log( + `Real Agent protocol E2E OK: general events=${generalEvents.length}, data actions=${actionOrder.length}, ` + + `handoff events=${handoffEvents.length}.` + ); +} + +async function runAgent(baseUrl, input) { + const response = await fetch(`${baseUrl}/api/copilotkit`, { + method: "POST", + headers: { + Accept: "text/event-stream", + Authorization: "Bearer dev-token", + "Content-Type": "application/json", + "X-Workspace-Id": "default" + }, + body: JSON.stringify({ + method: "agent/run", + params: { agentId: "dataFoundry" }, + body: { + threadId: input.threadId, + runId: input.runId, + state: {}, + messages: [{ id: `${input.runId}:user`, role: "user", content: input.prompt }], + tools: [], + context: [], + forwardedProps: { run_config: input.runConfig } + } + }), + signal: AbortSignal.timeout(5 * 60 * 1000) + }); + if (!response.ok) { + throw new Error(`Agent API failed (${response.status}): ${await response.text()}`); + } + return parseEventStream(await response.text()); +} + +const parseEventStream = (text) => text + .split("\n\n") + .map((chunk) => chunk.trim()) + .filter((chunk) => chunk.startsWith("data: ")) + .map((chunk) => chunk.slice("data: ".length)) + .filter((chunk) => chunk !== "[DONE]") + .map((chunk) => JSON.parse(chunk)); + +const protocolEvent = (events, name) => events.find((event) => event.type === "CUSTOM" && event.name === name); + +const toolResults = (events, toolName) => { + const names = new Map(events + .filter((event) => event.type === "TOOL_CALL_START" && typeof event.toolCallId === "string") + .map((event) => [event.toolCallId, event.toolCallName])); + return events.filter((event) => event.type === "TOOL_CALL_RESULT" + && (event.toolCallName ?? names.get(event.toolCallId)) === toolName); +}; + +const parseClaims = (content) => { + if (typeof content !== "string") return []; + try { + const parsed = JSON.parse(content); + return Array.isArray(parsed?.claims) ? parsed.claims : []; + } catch { + return []; + } +}; + +const assertProtocolTerminal = (events, terminalName) => { + const protocolTerminals = events.filter((event) => event.type === "CUSTOM" && event.name?.startsWith("protocol.run.")) + .map((event) => ({ name: event.name, decision: event.value?.payload?.decision })); + const runError = events.findLast((event) => event.type === "RUN_ERROR"); + assert.ok( + protocolEvent(events, terminalName), + `Missing ${terminalName}; terminals=${JSON.stringify(protocolTerminals)}; runError=${JSON.stringify(runError)}` + ); + assert.equal(events.findLast((event) => event.type === "RUN_ERROR"), undefined); + assert.equal(events.findLast((event) => event.type === "RUN_FINISHED")?.type, "RUN_FINISHED"); +}; + +if (process.env.PROTOCOL_E2E_API_URL) { + await runApiScenarios(process.env.PROTOCOL_E2E_API_URL.replace(/\/$/u, "")); +} +process.exit(0); diff --git a/scripts/smoke-collaboration-tools.mjs b/scripts/smoke-collaboration-tools.mjs index bf7b552..d62ddba 100644 --- a/scripts/smoke-collaboration-tools.mjs +++ b/scripts/smoke-collaboration-tools.mjs @@ -74,7 +74,7 @@ try { }; const runtime = new InteractionRuntimeAdapter(store, userId, sessionId, runId); const requested = runtime.capture(createCustomEvent("on_interrupt", JSON.stringify(interrupt))); - assert(requested?.name === "interaction.requested", "interrupt should project to interaction.requested"); + assert(requested?.event.name === "interaction.requested", "interrupt should project to interaction.requested"); const stored = store.interactions.getByToolCall({ user_id: userId, run_id: runId, diff --git a/scripts/smoke-datalink-semantic.mjs b/scripts/smoke-datalink-semantic.mjs new file mode 100644 index 0000000..988c2d2 --- /dev/null +++ b/scripts/smoke-datalink-semantic.mjs @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; + +const url = process.env.DATALINK_MCP_URL ?? "http://1.95.190.111:8080/mcp"; +const client = new Client({ name: "datafoundry-semantic-contract-smoke", version: "0.1.0" }); + +try { + await client.connect(new StreamableHTTPClientTransport(new URL(url))); + const listed = await client.listTools(); + const explore = listed.tools.find((tool) => tool.name === "datalink_explore"); + assert.ok(explore, "datalink_explore must be exposed"); + assert.deepEqual(explore.inputSchema.required, ["query"]); + assert.equal(Object.hasOwn(explore.inputSchema.properties ?? {}, "dataset"), false); + assert.equal(Object.hasOwn(explore.inputSchema.properties ?? {}, "mask_credential"), true); + + const result = await client.callTool({ + name: "datalink_explore", + arguments: { + query: "schema relationships", + max_nodes: 2, + focus: "schema", + mask_credential: true + } + }); + assert.equal(result.isError, false); + assert.ok(Array.isArray(result.content) && result.content.length > 0); + console.log(`DataLink semantic contract smoke OK: ${listed.tools.length} tools at ${url}.`); +} finally { + await client.close().catch(() => undefined); +} diff --git a/scripts/smoke-protocol-recovery.mjs b/scripts/smoke-protocol-recovery.mjs new file mode 100644 index 0000000..150e09d --- /dev/null +++ b/scripts/smoke-protocol-recovery.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createMetadataStore } from "../packages/metadata/dist/index.js"; + +const root = mkdtempSync(join(tmpdir(), "datafoundry-protocol-recovery-")); +const databasePath = join(root, "metadata.sqlite"); +const userId = "dev-user"; +const sessionId = "protocol-session"; +const runId = "protocol-run"; + +try { + let store = createMetadataStore({ database_path: databasePath }); + store.sessions.create({ user_id: userId, id: sessionId, title: "Protocol recovery" }); + store.runs.create({ + user_id: userId, + id: runId, + session_id: sessionId, + user_input: "Analyze orders", + status: "running" + }); + store.contextPackageSnapshots.create({ + user_id: userId, + session_id: sessionId, + run_id: runId, + package_id: "context-1", + revision: 0, + payload: { packageId: "context-1", revision: 0 } + }); + const initialState = createState(0, "semantic_grounding"); + store.protocolStates.compareAndSet({ + user_id: userId, + run_id: runId, + segment_id: "segment-1", + expected_revision: -1, + state: initialState + }); + + assert.throws(() => store.protocolStates.compareAndSet({ + user_id: userId, + run_id: runId, + segment_id: "segment-1", + expected_revision: -1, + state: initialState + }), /PROTOCOL_REVISION_CONFLICT:protocol-run:segment-1:-1:0/); + + const stateUpdatedEvent = { + eventId: "segment-1:1:protocol.state.updated", + type: "protocol.state.updated", + runId, + segmentId: "segment-1", + protocolId: "data-analysis", + protocolVersion: "1", + revision: 1 + }; + store.protocolStates.compareAndSetWithEvents({ + user_id: userId, + run_id: runId, + segment_id: "segment-1", + expected_revision: 0, + state: createState(1, "execution") + }, [stateUpdatedEvent]); + store.close(); + + store = createMetadataStore({ database_path: databasePath }); + const recovered = store.protocolStates.latestByRun({ user_id: userId, run_id: runId }); + assert.equal(recovered?.revision, 1); + assert.equal(JSON.parse(recovered?.state_json ?? "{}").phase, "execution"); + assert.deepEqual(store.protocolStates.pendingEvents({ user_id: userId, run_id: runId }), [stateUpdatedEvent]); + store.protocolStates.acknowledgeEvent({ user_id: userId, event_id: stateUpdatedEvent.eventId }); + assert.deepEqual(store.protocolStates.pendingEvents({ user_id: userId, run_id: runId }), []); + assert.throws(() => store.protocolStates.compareAndSet({ + user_id: userId, + run_id: runId, + segment_id: "segment-missing-context", + expected_revision: -1, + state: { + ...createState(0, "semantic_grounding"), + segmentId: "segment-missing-context", + contextPackageRef: { packageId: "missing-context", revision: 0 } + } + }), /PROTOCOL_CONTEXT_REF_NOT_FOUND:missing-context@0/); + store.close(); + + console.log("Protocol recovery smoke OK: CAS, event journal, restart recovery, and ContextPackage refs verified."); +} finally { + rmSync(root, { recursive: true, force: true }); +} + +function createState(revision, phase) { + return { + protocolId: "data-analysis", + protocolVersion: "1", + runId, + segmentId: "segment-1", + revision, + phase, + status: "active", + contextPackageRef: { packageId: "context-1", revision: 0 }, + actions: [], + completionRejections: 0, + domain: {} + }; +} diff --git a/scripts/smoke-run-finalizer.mjs b/scripts/smoke-run-finalizer.mjs index 67cad57..8c81f77 100644 --- a/scripts/smoke-run-finalizer.mjs +++ b/scripts/smoke-run-finalizer.mjs @@ -16,6 +16,7 @@ try { await runCanceledDraftScenario(metadataStore); await runCanceledEmptyDraftScenario(metadataStore); + await runCompletionRequiresDecisionScenario(metadataStore); await runCompletedNoDuplicateScenario(metadataStore); console.log("Run finalizer smoke OK: canceled runs persist assistant drafts without completion memory."); @@ -84,6 +85,11 @@ async function runCompletedNoDuplicateScenario(metadataStore) { const finalizer = createFinalizer(metadataStore, sessionId, runId, observer, []); await finalizer.complete({ + terminalDecision: { + status: "completed", + evaluatedContextPackageRef: { packageId: "context-complete", revision: 1 }, + evidenceRefs: [] + }, terminalEvent: { type: EventType.RUN_FINISHED, timestamp: Date.now() }, }); @@ -96,6 +102,19 @@ async function runCompletedNoDuplicateScenario(metadataStore) { assert.equal(metadataStore.runs.get({ user_id: userId, run_id: runId }).status, "completed"); } +async function runCompletionRequiresDecisionScenario(metadataStore) { + const sessionId = "complete-requires-decision-session"; + const runId = "complete-requires-decision-run"; + const observer = createObservedRun(metadataStore, sessionId, runId, "Complete only when governed"); + const finalizer = createFinalizer(metadataStore, sessionId, runId, observer, []); + + await assert.rejects( + finalizer.complete({ terminalEvent: { type: EventType.RUN_FINISHED, timestamp: Date.now() } }), + /PROTOCOL_TERMINAL_DECISION_REQUIRED/ + ); + assert.equal(metadataStore.runs.get({ user_id: userId, run_id: runId }).status, "running"); +} + function createObservedRun(metadataStore, sessionId, runId, userInput) { metadataStore.sessions.create({ user_id: userId, id: sessionId, title: sessionId }); metadataStore.runs.create({