From eaa8d4b1a519492444e9d5b63afe642fd95a9622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elberte=20Pl=C3=ADnio?= Date: Mon, 13 Jul 2026 14:58:23 -0300 Subject: [PATCH] feat(evidence): render action reports Co-Authored-By: Claude --- docs/releases/UNRELEASED.md | 19 ++ packages/cli/src/commands/artifacts.ts | 40 +-- packages/cli/test/commands.test.ts | 69 +++++ packages/core/src/evidence-render.ts | 297 +++++++++++++++++++++ packages/core/src/evidence.ts | 68 +++-- packages/core/src/index.ts | 9 + packages/core/test/evidence-render.test.ts | 252 +++++++++++++++++ packages/core/test/evidence.test.ts | 16 ++ packages/mcp-server/src/resources.ts | 237 ++++++++++++++-- packages/mcp-server/src/tools/artifacts.ts | 42 +-- packages/mcp-server/test/resources.test.ts | 202 +++++++++++++- packages/mcp-server/test/tools.test.ts | 47 ++++ 12 files changed, 1190 insertions(+), 108 deletions(-) create mode 100644 packages/core/src/evidence-render.ts create mode 100644 packages/core/test/evidence-render.test.ts diff --git a/docs/releases/UNRELEASED.md b/docs/releases/UNRELEASED.md index 32c933b..cb6cf99 100644 --- a/docs/releases/UNRELEASED.md +++ b/docs/releases/UNRELEASED.md @@ -113,6 +113,18 @@ then reset this file. unaffected. Config resolution gained `evidence.enabled` (product configuration, default true) via `isEvidenceEnabled`. No MCP or DevTools relay instrumentation is included. +- Added backward-compatible evidence rendering for recorded runs: CLI and MCP + artifact reports now include a deterministically ordered, defensively redacted + action timeline, while legacy runs remain artifact inventories. Evidence runs + can publish an escaped, no-script `report.html` filmstrip with a restrictive + CSP and only confined regular screenshot files. MCP resources expose sorted + action JSON and the static HTML report while rejecting legacy, malformed, + missing, symlinked, swapped, or out-of-run evidence files. Fixed-file resource + reads bind no-follow descriptors to the validated file identity and re-check + the pathname after reading; journal reads/appends also use no-follow opens. + Report publication uses an exclusive temporary file plus atomic rename so + planted symlinks are never followed. No action producers + or DevTools/MCP instrumentation are included in this slice. - Hosted CI now installs `x11vnc` alongside the other desktop test dependencies, and the desktop-linux integration suite asserts `x11vnc` is present when `CI=true` so VNC tests fail loudly instead of silently @@ -256,6 +268,13 @@ then reset this file. resisting spoofed-`runId`/fake-inflation manifests, and old-manifest list/read compatibility, are also covered. `bun run test:coverage` passes global thresholds. +- Evidence rendering/resources: `bun run typecheck` and `bun run build` pass; + the full and coverage suites pass 74 files / 916 passed / 2 skipped. Focused + tests cover deterministic ordering and post-sort step numbers, legacy reports, + corrupt/torn journals, planted secrets, HTML/XSS escaping and CSP, confined + screenshot embedding, action/report resource MIME and listings, and rejection + of symlinked journals, reports, screenshots, run directories, planted report + publication targets, and parent-directory swaps between validation and open. ### Not tested yet diff --git a/packages/cli/src/commands/artifacts.ts b/packages/cli/src/commands/artifacts.ts index 5022f7e..3ee9ed8 100644 --- a/packages/cli/src/commands/artifacts.ts +++ b/packages/cli/src/commands/artifacts.ts @@ -1,6 +1,13 @@ import { spawn } from "node:child_process"; import path from "node:path"; -import { listRuns, runsDir, type RunManifest } from "@pickforge/picklab-core"; +import { + isEvidenceRun, + listRuns, + readActions, + renderRunReport, + runsDir, + type RunManifest, +} from "@pickforge/picklab-core"; import { findOnPath } from "@pickforge/picklab-desktop-linux"; import { resolveProjectDir, @@ -88,34 +95,6 @@ export async function runArtifactsOpen( }); } -export function renderRunReport(manifest: RunManifest, dir: string): string[] { - const lines = [ - `# PickLab run ${manifest.runId}`, - "", - `- Slug: ${manifest.slug}`, - `- Status: ${manifest.status}`, - `- Created: ${manifest.createdAt}`, - ]; - if (manifest.sessionId !== undefined) { - lines.push(`- Session: ${manifest.sessionId}`); - } - lines.push( - `- Directory: ${dir}`, - "", - `## Artifacts (${manifest.artifacts.length})`, - "", - ); - if (manifest.artifacts.length === 0) { - lines.push("(none)"); - } - for (const artifact of manifest.artifacts) { - lines.push( - `- [${artifact.type}] ${artifact.name} — ${artifact.path} (${artifact.createdAt})`, - ); - } - return lines; -} - export async function runArtifactsReport( runId: string | undefined, opts: BaseCliOptions, @@ -123,9 +102,10 @@ export async function runArtifactsReport( return runReported(opts, async () => { const projectDir = resolveProjectDir(opts); const { manifest, dir } = await findRun(projectDir, runId); + const records = isEvidenceRun(manifest) ? await readActions(dir) : []; return { data: { runId: manifest.runId, dir, manifest }, - lines: renderRunReport(manifest, dir), + lines: renderRunReport(manifest, dir, records), }; }); } diff --git a/packages/cli/test/commands.test.ts b/packages/cli/test/commands.test.ts index 114c8b1..99d9dc1 100644 --- a/packages/cli/test/commands.test.ts +++ b/packages/cli/test/commands.test.ts @@ -1406,7 +1406,36 @@ describe("picklab artifacts", () => { }); writeSyntheticRun(projectDir, "20260609-120000-synthetic", { sessionId: "desk-12345678", + evidenceVersion: 1, + actionLog: "actions.jsonl", }); + fs.writeFileSync( + path.join( + projectDir, + ".picklab", + "runs", + "20260609-120000-synthetic", + "actions.jsonl", + ), + [ + JSON.stringify({ + actionId: "second", + source: "mcp", + tool: "desktop_type", + startedAt: "2026-06-09T12:00:04.000Z", + status: "error", + error: `Authorization: Bearer ${PLANTED_TOKEN}`, + }), + JSON.stringify({ + actionId: "first", + source: "mcp", + tool: "desktop_click", + startedAt: "2026-06-09T12:00:03.000Z", + status: "ok", + }), + "", + ].join("\n"), + ); const result = await runCli( ["artifacts", "report", "--project-dir", projectDir], @@ -1419,6 +1448,11 @@ describe("picklab artifacts", () => { expect(result.stdout).toContain( "- [screenshot] screenshot.png — screenshots/screenshot.png", ); + expect(result.stdout.indexOf("Step 1 — mcp / desktop_click")).toBeLessThan( + result.stdout.indexOf("Step 2 — mcp / desktop_type"), + ); + expect(result.stdout).toContain("[REDACTED]"); + expect(result.stdout).not.toContain(PLANTED_TOKEN); const specific = await runCli( [ @@ -1437,6 +1471,41 @@ describe("picklab artifacts", () => { expect(report.dir).toBe( path.join(projectDir, ".picklab", "runs", "20260609-110000-synthetic"), ); + + const legacy = await runCli( + [ + "artifacts", + "report", + "20260609-110000-synthetic", + "--project-dir", + projectDir, + ], + env, + ); + expect(legacy.code).toBe(0); + expect(legacy.stdout).not.toContain("## Actions"); + }); + + it("fails actionably for a corrupt evidence journal", async () => { + const env = makeEnv(); + const projectDir = makeProjectDir(); + const runId = "20260609-120000-corrupt"; + writeSyntheticRun(projectDir, runId, { + evidenceVersion: 1, + actionLog: "actions.jsonl", + }); + fs.writeFileSync( + path.join(projectDir, ".picklab", "runs", runId, "actions.jsonl"), + '{"actionId":"ok"}\nnot-json\n', + ); + + const result = await runCli( + ["artifacts", "report", runId, "--project-dir", projectDir, "--json"], + env, + ); + expect(result.code).toBe(1); + const report = parseJson(result); + expect(report.errors[0]).toContain("Corrupt evidence journal"); }); it("fails for unknown run ids", async () => { diff --git a/packages/core/src/evidence-render.ts b/packages/core/src/evidence-render.ts new file mode 100644 index 0000000..5ace61f --- /dev/null +++ b/packages/core/src/evidence-render.ts @@ -0,0 +1,297 @@ +import fs from "node:fs"; +import path from "node:path"; +import { redactSecrets } from "./redact.js"; +import type { RunManifest } from "./run.js"; +import { + isEvidenceRun, + isTruncationRecord, + readActions, + type EvidenceAction, + type EvidenceRecord, +} from "./evidence.js"; + +export const EVIDENCE_REPORT = "report.html"; + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function recordTimestamp(record: EvidenceRecord): string { + return isTruncationRecord(record) ? record.recordedAt : record.startedAt; +} + +export function sortEvidenceRecords( + records: readonly EvidenceRecord[], +): EvidenceRecord[] { + return records + .map((record, index) => ({ record, index })) + .sort((left, right) => { + const byTime = compareText( + recordTimestamp(left.record), + recordTimestamp(right.record), + ); + if (byTime !== 0) return byTime; + const byId = compareText(left.record.actionId, right.record.actionId); + return byId !== 0 ? byId : left.index - right.index; + }) + .map(({ record }) => record); +} + +function stableValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stableValue); + if (typeof value !== "object" || value === null) return value; + const result: Record = {}; + for (const key of Object.keys(value).sort(compareText)) { + result[key] = stableValue((value as Record)[key]); + } + return result; +} + +function safeText(value: unknown): string { + return redactSecrets(String(value)); +} + +function stableJson(value: unknown): string { + return redactSecrets(JSON.stringify(stableValue(value))); +} + +function actionTitle(action: EvidenceAction): string { + return `${safeText(action.source)} / ${safeText(action.tool)}`; +} + +export function renderRunReport( + manifest: RunManifest, + dir: string, + records: readonly EvidenceRecord[] = [], +): string[] { + const lines = [ + `# PickLab run ${safeText(manifest.runId)}`, + "", + `- Slug: ${safeText(manifest.slug)}`, + `- Status: ${safeText(manifest.status)}`, + `- Created: ${safeText(manifest.createdAt)}`, + ]; + if (manifest.sessionId !== undefined) { + lines.push(`- Session: ${safeText(manifest.sessionId)}`); + } + lines.push( + `- Directory: ${safeText(dir)}`, + "", + `## Artifacts (${manifest.artifacts.length})`, + "", + ); + if (manifest.artifacts.length === 0) { + lines.push("(none)"); + } + for (const artifact of manifest.artifacts) { + lines.push( + `- [${safeText(artifact.type)}] ${safeText(artifact.name)} — ` + + `${safeText(artifact.path)} (${safeText(artifact.createdAt)})`, + ); + } + + if (!isEvidenceRun(manifest)) return lines; + + const ordered = sortEvidenceRecords(records); + lines.push("", `## Actions (${ordered.length})`, ""); + if (ordered.length === 0) { + lines.push("(none)"); + return lines; + } + ordered.forEach((record, index) => { + const step = index + 1; + if (isTruncationRecord(record)) { + lines.push( + `### Step ${step} — Evidence truncated`, + "", + `- Recorded: ${safeText(record.recordedAt)}`, + `- Bytes: ${record.bytes} / ${record.maxBytes}`, + "", + ); + return; + } + lines.push( + `### Step ${step} — ${actionTitle(record)}`, + "", + `- Started: ${safeText(record.startedAt)}`, + `- Status: ${safeText(record.status)}`, + ); + if (record.sessionId !== undefined) { + lines.push(`- Session: ${safeText(record.sessionId)}`); + } + if (record.durationMs !== undefined) { + lines.push(`- Duration: ${record.durationMs} ms`); + } + if (record.target !== undefined) { + lines.push(`- Target: ${stableJson(record.target)}`); + } + if (record.artifacts !== undefined && record.artifacts.length > 0) { + lines.push( + `- Artifacts: ${record.artifacts.map(safeText).join(", ")}`, + ); + } + if (record.error !== undefined) { + lines.push(`- Error: ${safeText(record.error)}`); + } + lines.push(""); + }); + return lines; +} + +function escapeHtml(value: unknown): string { + return safeText(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function renderMetadata(label: string, value: unknown): string { + return `
${escapeHtml(label)}
${escapeHtml(value)}
`; +} + +function safeScreenshotPath(value: string): boolean { + return /^screenshots\/[A-Za-z0-9._-]+\.png$/.test(value) && !value.includes(".."); +} + +function renderAction( + action: EvidenceAction, + step: number, + safeScreenshots: ReadonlySet, +): string { + const metadata = [ + renderMetadata("Started", action.startedAt), + renderMetadata("Status", action.status), + ]; + if (action.sessionId !== undefined) { + metadata.push(renderMetadata("Session", action.sessionId)); + } + if (action.durationMs !== undefined) { + metadata.push(renderMetadata("Duration", `${action.durationMs} ms`)); + } + if (action.target !== undefined) { + metadata.push(renderMetadata("Target", stableJson(action.target))); + } + if (action.error !== undefined) { + metadata.push(renderMetadata("Error", action.error)); + } + + const screenshots = (action.artifacts ?? []) + .filter((artifact) => safeScreenshots.has(artifact)) + .map( + (artifact) => + `
Screenshot for step ${step}
${escapeHtml(artifact)}
`, + ) + .join(""); + + return `
+
Step ${step}

${escapeHtml(actionTitle(action))}

+
${metadata.join("")}
+${screenshots === "" ? "" : `
${screenshots}
`} +
`; +} + +export function renderEvidenceHtml( + manifest: RunManifest, + records: readonly EvidenceRecord[], + safeScreenshots: ReadonlySet = new Set(), +): string { + const ordered = sortEvidenceRecords(records); + const steps = ordered + .map((record, index) => { + const step = index + 1; + if (!isTruncationRecord(record)) { + return renderAction(record, step, safeScreenshots); + } + return `
+
Step ${step}

Evidence truncated

+
${renderMetadata("Recorded", record.recordedAt)}${renderMetadata("Bytes", `${record.bytes} / ${record.maxBytes}`)}
+
`; + }) + .join("\n"); + + return ` + + + + + +PickLab run ${escapeHtml(manifest.runId)} + + + +
+
+

PickLab run ${escapeHtml(manifest.runId)}

+
${renderMetadata("Slug", manifest.slug)}${renderMetadata("Status", manifest.status)}${renderMetadata("Created", manifest.createdAt)}${manifest.sessionId === undefined ? "" : renderMetadata("Session", manifest.sessionId)}
+
+
+${steps === "" ? '

No recorded actions.

' : steps} +
+
+ + +`; +} + +async function collectSafeScreenshots( + runDir: string, + records: readonly EvidenceRecord[], +): Promise> { + const safe = new Set(); + const realRunDir = await fs.promises.realpath(runDir); + const candidates = new Set( + records.flatMap((record) => + isTruncationRecord(record) ? [] : (record.artifacts ?? []), + ), + ); + for (const relative of candidates) { + if (!safeScreenshotPath(relative)) continue; + const candidate = path.join(runDir, relative); + try { + const stat = await fs.promises.lstat(candidate); + if (stat.isSymbolicLink() || !stat.isFile()) continue; + const realCandidate = await fs.promises.realpath(candidate); + if (realCandidate !== path.join(realRunDir, relative)) continue; + safe.add(relative); + } catch { + continue; + } + } + return safe; +} + +let reportTmpCounter = 0; + +export async function writeEvidenceReport( + runDir: string, + manifest: RunManifest, +): Promise { + if (!isEvidenceRun(manifest)) { + throw new Error(`Run ${manifest.runId} is not an evidence run`); + } + const runStat = await fs.promises.lstat(runDir); + if (runStat.isSymbolicLink() || !runStat.isDirectory()) { + throw new Error(`Unsafe evidence run directory: ${runDir}`); + } + const records = await readActions(runDir); + const safeScreenshots = await collectSafeScreenshots(runDir, records); + const html = renderEvidenceHtml(manifest, records, safeScreenshots); + const target = path.join(runDir, EVIDENCE_REPORT); + reportTmpCounter += 1; + const tmp = path.join( + runDir, + `.${EVIDENCE_REPORT}.tmp-${process.pid}-${reportTmpCounter}`, + ); + await fs.promises.writeFile(tmp, html, { encoding: "utf8", flag: "wx" }); + try { + await fs.promises.rename(tmp, target); + } catch (error) { + await fs.promises.unlink(tmp).catch(() => {}); + throw error; + } + return target; +} diff --git a/packages/core/src/evidence.ts b/packages/core/src/evidence.ts index 77d6b76..3cb471a 100644 --- a/packages/core/src/evidence.ts +++ b/packages/core/src/evidence.ts @@ -948,7 +948,14 @@ export async function appendAction( const journalPath = path.join(runDir, EVIDENCE_ACTION_LOG); return withJournalLock(runDir, async () => { - const handle = await fs.promises.open(journalPath, "a+"); + const handle = await fs.promises.open( + journalPath, + fs.constants.O_RDWR | + fs.constants.O_CREAT | + fs.constants.O_APPEND | + fs.constants.O_NOFOLLOW, + 0o600, + ); try { await repairTornJournalTail(handle); // Derive current usage from the run directory's real on-disk bytes (journal @@ -1297,23 +1304,11 @@ async function writeTruncationMarkerOnce( return false; } -/** - * Read the run's action journal deterministically. Records are returned in file - * (append) order. A missing/empty journal yields `[]`. Only a torn final line - * (an unterminated last record from an interrupted write) is tolerated and - * dropped; any malformed or blank line before the end is rejected. - */ -export async function readActions(runDir: string): Promise { - let raw: string; - try { - raw = await fs.promises.readFile( - path.join(runDir, EVIDENCE_ACTION_LOG), - "utf8", - ); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; - throw error; - } +/** Parse an action journal using the same torn-tail and corruption rules as reads. */ +export function parseActionsJournal( + raw: string, + journalLabel: string, +): EvidenceRecord[] { if (raw === "") return []; const segments = raw.split("\n"); @@ -1326,7 +1321,7 @@ export async function readActions(runDir: string): Promise { const segment = segments[index]!; if (segment === "") { throw new Error( - `Corrupt evidence journal in ${runDir}: blank record at line ${index + 1}`, + `Corrupt evidence journal in ${journalLabel}: blank record at line ${index + 1}`, ); } let parsed: unknown; @@ -1334,7 +1329,7 @@ export async function readActions(runDir: string): Promise { parsed = JSON.parse(segment); } catch (error) { throw new Error( - `Corrupt evidence journal in ${runDir} at line ${index + 1}: ` + + `Corrupt evidence journal in ${journalLabel} at line ${index + 1}: ` + `${(error as Error).message}`, ); } @@ -1344,7 +1339,7 @@ export async function readActions(runDir: string): Promise { typeof (parsed as EvidenceAction).actionId !== "string" ) { throw new Error( - `Corrupt evidence journal in ${runDir} at line ${index + 1}: ` + + `Corrupt evidence journal in ${journalLabel} at line ${index + 1}: ` + `record is not a valid evidence record`, ); } @@ -1353,6 +1348,37 @@ export async function readActions(runDir: string): Promise { return records; } +/** + * Read the run's action journal deterministically. Records are returned in file + * (append) order. A missing/empty journal yields `[]`. Only a torn final line + * (an unterminated last record from an interrupted write) is tolerated and + * dropped; any malformed or blank line before the end is rejected. + */ +export async function readActions(runDir: string): Promise { + const journalPath = path.join(runDir, EVIDENCE_ACTION_LOG); + let handle: fs.promises.FileHandle; + try { + handle = await fs.promises.open( + journalPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + let raw: string; + try { + const stat = await handle.stat(); + if (!stat.isFile()) { + throw new Error(`Unsafe evidence journal in ${runDir}: not a regular file`); + } + raw = await handle.readFile("utf8"); + } finally { + await handle.close(); + } + return parseActionsJournal(raw, runDir); +} + export function isEvidenceRun(manifest: RunManifest): boolean { return manifest.evidenceVersion === EVIDENCE_VERSION; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d73d982..dd71869 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -49,6 +49,7 @@ export { isEvidenceRun, isEvidenceTruncated, isTruncationRecord, + parseActionsJournal, pruneFinalizedEvidenceRuns, readActions, resolveActivePointer, @@ -67,6 +68,14 @@ export { type PruneEvidenceOptions, } from "./evidence.js"; +export { + EVIDENCE_REPORT, + renderEvidenceHtml, + renderRunReport, + sortEvidenceRecords, + writeEvidenceReport, +} from "./evidence-render.js"; + export { isSecretKey, redactEnv, redactSecrets } from "./redact.js"; export { diff --git a/packages/core/test/evidence-render.test.ts b/packages/core/test/evidence-render.test.ts new file mode 100644 index 0000000..36d5b1c --- /dev/null +++ b/packages/core/test/evidence-render.test.ts @@ -0,0 +1,252 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + EVIDENCE_REPORT, + appendAction, + createRun, + renderEvidenceHtml, + renderRunReport, + sortEvidenceRecords, + writeEvidenceReport, + type EvidenceAction, + type EvidenceRecord, + type RunManifest, +} from "../src/index.js"; + +const TOKEN = `ghp_${"a".repeat(36)}`; + +let root: string; +let projectDir: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "picklab-evidence-render-")); + projectDir = path.join(root, "project"); + fs.mkdirSync(projectDir, { recursive: true }); +}); + +afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); +}); + +function action(overrides: Partial = {}): EvidenceAction { + return { + actionId: "act-1", + source: "mcp", + tool: "desktop_click", + startedAt: "2026-07-13T12:00:00.000Z", + status: "ok", + ...overrides, + }; +} + +function evidenceManifest(overrides: Partial = {}): RunManifest { + return { + runId: "20260713-120000-evidence", + slug: "evidence", + createdAt: "2026-07-13T12:00:00.000Z", + status: "completed", + artifacts: [], + evidenceVersion: 1, + actionLog: "actions.jsonl", + ...overrides, + }; +} + +describe("sortEvidenceRecords", () => { + it("orders by timestamp then action id without mutating append order", () => { + const appended: EvidenceRecord[] = [ + action({ actionId: "z", startedAt: "2026-07-13T12:00:02.000Z" }), + action({ actionId: "b", startedAt: "2026-07-13T12:00:01.000Z" }), + action({ actionId: "a", startedAt: "2026-07-13T12:00:01.000Z" }), + { + actionId: "marker", + evidenceTruncated: true, + reason: "evidence-cap", + bytes: 100, + maxBytes: 100, + recordedAt: "2026-07-13T12:00:03.000Z", + }, + ]; + + expect(sortEvidenceRecords(appended).map((record) => record.actionId)).toEqual([ + "a", + "b", + "z", + "marker", + ]); + expect(appended.map((record) => record.actionId)).toEqual([ + "z", + "b", + "a", + "marker", + ]); + }); +}); + +describe("renderRunReport", () => { + it("keeps legacy reports as artifact inventories", () => { + const lines = renderRunReport( + { + runId: "legacy", + slug: "legacy", + createdAt: "2026-07-13T12:00:00.000Z", + status: "completed", + artifacts: [], + }, + "/tmp/legacy", + ); + + expect(lines).toContain("## Artifacts (0)"); + expect(lines.join("\n")).not.toContain("## Actions"); + }); + + it("renders a deterministic redacted action timeline", () => { + const lines = renderRunReport(evidenceManifest(), "/tmp/evidence", [ + action({ + actionId: "second", + tool: "desktop_type", + startedAt: "2026-07-13T12:00:02.000Z", + target: { z: 1, a: `token=${TOKEN}` }, + }), + action({ + actionId: "first", + startedAt: "2026-07-13T12:00:01.000Z", + }), + ]); + const report = lines.join("\n"); + + expect(report.indexOf("Step 1 — mcp / desktop_click")).toBeLessThan( + report.indexOf("Step 2 — mcp / desktop_type"), + ); + expect(report).toContain('Target: {"a":"token=[REDACTED]","z":1}'); + expect(report).not.toContain(TOKEN); + }); +}); + +describe("renderEvidenceHtml", () => { + it("escapes page-controlled text, redacts secrets, and makes no external requests", () => { + const html = renderEvidenceHtml( + evidenceManifest({ + runId: '', + slug: `token=${TOKEN}`, + }), + [ + action({ + source: '', + tool: "desktop_type", + target: { label: "" }, + error: `Authorization: Bearer ${TOKEN}`, + artifacts: [ + "screenshots/good.png", + "https://evil.invalid/leak.png", + "screenshots/../../leak.png", + ], + }), + ], + new Set(["screenshots/good.png"]), + ); + + expect(html).toContain("Content-Security-Policy"); + expect(html).toContain("default-src 'none'"); + expect(html).not.toContain(" { + const html = renderEvidenceHtml(evidenceManifest(), [ + action({ actionId: "later", startedAt: "2026-07-13T12:00:02.000Z" }), + action({ + actionId: "earlier", + tool: "desktop_move", + startedAt: "2026-07-13T12:00:01.000Z", + }), + ]); + + expect(html.indexOf("Step 1")).toBeLessThan(html.indexOf("desktop_move")); + expect(html.indexOf("desktop_move")).toBeLessThan(html.indexOf("Step 2")); + expect(html.indexOf("Step 2")).toBeLessThan(html.indexOf("desktop_click")); + }); +}); + +describe("writeEvidenceReport", () => { + it("writes a static filmstrip and embeds only regular confined screenshots", async () => { + const run = await createRun(projectDir, "filmstrip", { + evidence: true, + now: new Date("2026-07-13T12:00:00.000Z"), + }); + const good = path.join(run.dir, "screenshots", "good.png"); + const outside = path.join(root, "outside.png"); + fs.writeFileSync(good, "png"); + fs.writeFileSync(outside, "secret"); + fs.symlinkSync(outside, path.join(run.dir, "screenshots", "escape.png")); + await appendAction( + run.dir, + action({ + artifacts: [ + "screenshots/good.png", + "screenshots/escape.png", + "../outside.png", + ], + }), + ); + + const reportPath = await writeEvidenceReport(run.dir, run.manifest); + const html = fs.readFileSync(reportPath, "utf8"); + + expect(reportPath).toBe(path.join(run.dir, EVIDENCE_REPORT)); + expect(html).toContain('src="screenshots/good.png"'); + expect(html).not.toContain("escape.png"); + expect(html).not.toContain("../outside.png"); + expect( + fs.readdirSync(run.dir).some((name) => name.includes("report.html.tmp")), + ).toBe(false); + }); + + it("replaces a planted report symlink without touching its target", async () => { + const run = await createRun(projectDir, "report-link", { evidence: true }); + const outside = path.join(root, "outside-report.html"); + fs.writeFileSync(outside, "outside-secret"); + const target = path.join(run.dir, EVIDENCE_REPORT); + fs.symlinkSync(outside, target); + + await writeEvidenceReport(run.dir, run.manifest); + + expect(fs.lstatSync(target).isSymbolicLink()).toBe(false); + expect(fs.readFileSync(outside, "utf8")).toBe("outside-secret"); + }); + + it("rejects a symlinked run directory", async () => { + const run = await createRun(projectDir, "real-run", { evidence: true }); + const linked = path.join(root, "linked-run"); + fs.symlinkSync(run.dir, linked); + + await expect(writeEvidenceReport(linked, run.manifest)).rejects.toThrow( + /Unsafe evidence run directory/, + ); + }); + + it("rejects legacy runs and corrupt evidence journals", async () => { + const legacy = await createRun(projectDir, "legacy"); + await expect(writeEvidenceReport(legacy.dir, legacy.manifest)).rejects.toThrow( + /not an evidence run/, + ); + + const evidence = await createRun(projectDir, "corrupt", { evidence: true }); + fs.writeFileSync( + path.join(evidence.dir, "actions.jsonl"), + '{"actionId":"ok"}\nnot-json\n', + ); + await expect( + writeEvidenceReport(evidence.dir, evidence.manifest), + ).rejects.toThrow(/Corrupt evidence journal/); + }); +}); diff --git a/packages/core/test/evidence.test.ts b/packages/core/test/evidence.test.ts index 6c4386c..7ca7eb2 100644 --- a/packages/core/test/evidence.test.ts +++ b/packages/core/test/evidence.test.ts @@ -457,6 +457,22 @@ describe("appendAction and readActions", () => { const plain = await createRun(project, "plain"); expect(await readActions(plain.dir)).toEqual([]); }); + + it("never follows a symlinked action journal", async () => { + const { run } = await beginEvidenceRun(project, "desk-link00"); + const journal = path.join(run.dir, "actions.jsonl"); + const outside = path.join(project, "outside-actions.jsonl"); + const outsideBody = `${JSON.stringify(action({ actionId: "outside" }))}\n`; + await fs.promises.writeFile(outside, outsideBody); + await fs.promises.rm(journal); + await fs.promises.symlink(outside, journal); + + await expect(readActions(run.dir)).rejects.toThrow(); + await expect( + appendAction(run.dir, action({ actionId: "blocked" })), + ).rejects.toThrow(); + expect(await fs.promises.readFile(outside, "utf8")).toBe(outsideBody); + }); }); describe("readActions corruption handling", () => { diff --git a/packages/mcp-server/src/resources.ts b/packages/mcp-server/src/resources.ts index a1d6e4c..331918a 100644 --- a/packages/mcp-server/src/resources.ts +++ b/packages/mcp-server/src/resources.ts @@ -6,11 +6,16 @@ import { } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { Variables } from "@modelcontextprotocol/sdk/shared/uriTemplate.js"; import { + EVIDENCE_ACTION_LOG, + EVIDENCE_REPORT, getSession, + isEvidenceRun, listRuns, listSessions, + parseActionsJournal, redactSecrets, runsDir, + sortEvidenceRecords, } from "@pickforge/picklab-core"; import type { ServerContext } from "./context.js"; import { isSafeRunId } from "./tools/artifacts.js"; @@ -131,47 +136,101 @@ async function assertWithinSubdir( } } -// Reject a manifest whose real location escapes the run dir via symlinks. When -// the file (or run dir) does not exist, return so the caller's read produces -// its usual not-found error. -async function assertManifestWithinRun( +// Reject a missing fixed run file, a symlink, or a real location that escapes +// the run directory. +async function assertRootFileWithinRun( ctx: ServerContext, runId: string, - manifestPath: string, + fileName: string, + filePath: string, notFound: () => Error, -): Promise { +): Promise { if (!(await isRunDirSafe(ctx, runId))) { throw notFound(); } + let expectedEntry: fs.Stats; + try { + expectedEntry = await fs.promises.lstat(filePath); + if (expectedEntry.isSymbolicLink() || !expectedEntry.isFile()) { + throw notFound(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === undefined) { + throw error; + } + throw notFound(); + } const runDir = path.join(runsDir(ctx.projectDir), runId); let realRunDir: string; let realFile: string; try { realRunDir = await fs.promises.realpath(runDir); - realFile = await fs.promises.realpath(manifestPath); + realFile = await fs.promises.realpath(filePath); } catch { - return; + throw notFound(); + } + if (realFile !== path.join(realRunDir, fileName)) { + throw notFound(); + } + return expectedEntry; +} + +function sameFileIdentity(left: fs.Stats, right: fs.Stats): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +// Open the validated file without following its final component, bind the +// descriptor to the validated inode, and reject a pathname swap during read. +async function readTextFileNoFollow( + filePath: string, + expected: fs.Stats, + notFound: () => Error, +): Promise { + let handle: fs.promises.FileHandle; + try { + handle = await fs.promises.open( + filePath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW, + ); + } catch { + throw notFound(); } - if (realFile !== path.join(realRunDir, "manifest.json")) { + try { + const opened = await handle.stat(); + if (!opened.isFile() || !sameFileIdentity(opened, expected)) { + throw notFound(); + } + const raw = await handle.readFile("utf8"); + const current = await fs.promises.lstat(filePath); + if ( + current.isSymbolicLink() || + !current.isFile() || + !sameFileIdentity(opened, current) + ) { + throw notFound(); + } + return raw; + } catch { throw notFound(); + } finally { + await handle.close(); } } -// Return true when a run's manifest.json passes the same realpath confinement -// as direct manifest reads (run dir not a symlink, manifest not a symlink -// escaping the run dir). Runs that fail this are excluded from listings so a -// symlinked manifest cannot leak data via resource enumeration. -async function isManifestSafe( +async function isRootFileSafe( ctx: ServerContext, runId: string, + fileName: string, ): Promise { if (!(await isRunDirSafe(ctx, runId))) return false; const runDir = path.join(runsDir(ctx.projectDir), runId); - const manifestPath = path.join(runDir, "manifest.json"); + const filePath = path.join(runDir, fileName); try { + const entry = await fs.promises.lstat(filePath); + if (entry.isSymbolicLink() || !entry.isFile()) return false; const realRunDir = await fs.promises.realpath(runDir); - const realFile = await fs.promises.realpath(manifestPath); - return realFile === path.join(realRunDir, "manifest.json"); + const realFile = await fs.promises.realpath(filePath); + return realFile === path.join(realRunDir, fileName); } catch { return false; } @@ -185,12 +244,25 @@ async function listSafeRuns( const safe: Awaited> = []; for (const manifest of await listRuns(ctx.projectDir)) { if (!isSafeRunId(manifest.runId)) continue; - if (!(await isManifestSafe(ctx, manifest.runId))) continue; + if (!(await isRootFileSafe(ctx, manifest.runId, "manifest.json"))) continue; safe.push(manifest); } return safe; } +async function listEvidenceRunFiles( + ctx: ServerContext, + fileName: typeof EVIDENCE_ACTION_LOG | typeof EVIDENCE_REPORT, +): Promise { + const runIds: string[] = []; + for (const manifest of await listSafeRuns(ctx)) { + if (!isEvidenceRun(manifest)) continue; + if (!(await isRootFileSafe(ctx, manifest.runId, fileName))) continue; + runIds.push(manifest.runId); + } + return runIds; +} + async function listRunFiles( ctx: ServerContext, subdir: "screenshots" | "logs", @@ -280,21 +352,138 @@ export function registerResources(server: McpServer, ctx: ServerContext): void { runId, "manifest.json", ); - await assertManifestWithinRun( + const expected = await assertRootFileWithinRun( ctx, runId, + "manifest.json", manifestPath, () => new Error(`Run not found: ${runId}`), ); - let raw: string; + const raw = await readTextFileNoFollow( + manifestPath, + expected, + () => new Error(`Run not found: ${runId}`), + ); + return { + contents: [ + { uri: uri.href, mimeType: "application/json", text: raw }, + ], + }; + }, + ); + + server.registerResource( + "run-actions", + new ResourceTemplate("picklab://runs/{runId}/actions", { + list: async () => ({ + resources: (await listEvidenceRunFiles(ctx, EVIDENCE_ACTION_LOG)).map( + (runId) => ({ + uri: `picklab://runs/${runId}/actions`, + name: `Run ${runId} actions`, + mimeType: "application/json", + }), + ), + }), + }), + { + title: "Run actions", + description: "Deterministically ordered evidence actions for a run", + mimeType: "application/json", + }, + async (uri, variables) => { + const runId = decodeVariable(variables, "runId"); + const manifest = (await listSafeRuns(ctx)).find( + (candidate) => candidate.runId === runId, + ); + if (manifest === undefined || !isEvidenceRun(manifest)) { + throw new Error(`Actions not found: ${runId}`); + } + const runDir = path.join(runsDir(ctx.projectDir), runId); + const actionsPath = path.join(runDir, EVIDENCE_ACTION_LOG); + const expected = await assertRootFileWithinRun( + ctx, + runId, + EVIDENCE_ACTION_LOG, + actionsPath, + () => new Error(`Actions not found: ${runId}`), + ); + let records; try { - raw = await fs.promises.readFile(manifestPath, "utf8"); - } catch { - throw new Error(`Run not found: ${runId}`); + const raw = await readTextFileNoFollow( + actionsPath, + expected, + () => new Error(`Actions not found: ${runId}`), + ); + records = parseActionsJournal(raw, `run ${runId}`); + } catch (error) { + throw new Error( + `Could not read actions for ${runId}: ${(error as Error).message}`, + ); } return { contents: [ - { uri: uri.href, mimeType: "application/json", text: raw }, + { + uri: uri.href, + mimeType: "application/json", + text: redactSecrets( + JSON.stringify(sortEvidenceRecords(records), null, 2), + ), + }, + ], + }; + }, + ); + + server.registerResource( + "run-report", + new ResourceTemplate("picklab://runs/{runId}/report", { + list: async () => ({ + resources: (await listEvidenceRunFiles(ctx, EVIDENCE_REPORT)).map( + (runId) => ({ + uri: `picklab://runs/${runId}/report`, + name: `Run ${runId} HTML report`, + mimeType: "text/html", + }), + ), + }), + }), + { + title: "Run HTML report", + description: "Static evidence filmstrip for a recorded run", + mimeType: "text/html", + }, + async (uri, variables) => { + const runId = decodeVariable(variables, "runId"); + const manifest = (await listSafeRuns(ctx)).find( + (candidate) => candidate.runId === runId, + ); + if (manifest === undefined || !isEvidenceRun(manifest)) { + throw new Error(`Report not found: ${runId}`); + } + const reportPath = path.join( + runsDir(ctx.projectDir), + runId, + EVIDENCE_REPORT, + ); + const expected = await assertRootFileWithinRun( + ctx, + runId, + EVIDENCE_REPORT, + reportPath, + () => new Error(`Report not found: ${runId}`), + ); + const html = await readTextFileNoFollow( + reportPath, + expected, + () => new Error(`Report not found: ${runId}`), + ); + return { + contents: [ + { + uri: uri.href, + mimeType: "text/html", + text: redactSecrets(html), + }, ], }; }, diff --git a/packages/mcp-server/src/tools/artifacts.ts b/packages/mcp-server/src/tools/artifacts.ts index e2d39f2..39a8547 100644 --- a/packages/mcp-server/src/tools/artifacts.ts +++ b/packages/mcp-server/src/tools/artifacts.ts @@ -1,7 +1,14 @@ import path from "node:path"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; -import { listRuns, runsDir, type RunManifest } from "@pickforge/picklab-core"; +import { + isEvidenceRun, + listRuns, + readActions, + renderRunReport, + runsDir, + type RunManifest, +} from "@pickforge/picklab-core"; import { runTool, type ServerContext } from "../context.js"; const RUN_ID_PATTERN = /^[A-Za-z0-9._-]+$/; @@ -39,34 +46,6 @@ export async function findRun( return { manifest, dir: path.join(runsDir(projectDir), manifest.runId) }; } -function renderRunReport(manifest: RunManifest, dir: string): string { - const lines = [ - `# PickLab run ${manifest.runId}`, - "", - `- Slug: ${manifest.slug}`, - `- Status: ${manifest.status}`, - `- Created: ${manifest.createdAt}`, - ]; - if (manifest.sessionId !== undefined) { - lines.push(`- Session: ${manifest.sessionId}`); - } - lines.push( - `- Directory: ${dir}`, - "", - `## Artifacts (${manifest.artifacts.length})`, - "", - ); - if (manifest.artifacts.length === 0) { - lines.push("(none)"); - } - for (const artifact of manifest.artifacts) { - lines.push( - `- [${artifact.type}] ${artifact.name} — ${artifact.path} (${artifact.createdAt})`, - ); - } - return lines.join("\n"); -} - export function registerArtifactTools( server: McpServer, ctx: ServerContext, @@ -100,7 +79,7 @@ export function registerArtifactTools( title: "Run report", description: "Render a report for one run (default: the most recent run), " + - "including its artifact inventory.", + "including its artifact inventory and evidence action timeline.", inputSchema: { runId: z.string().min(1).optional().describe("Run id"), }, @@ -108,12 +87,13 @@ export function registerArtifactTools( (args) => runTool(async () => { const { manifest, dir } = await findRun(ctx.projectDir, args.runId); + const records = isEvidenceRun(manifest) ? await readActions(dir) : []; return { data: { runId: manifest.runId, dir, manifest, - report: renderRunReport(manifest, dir), + report: renderRunReport(manifest, dir, records).join("\n"), }, }; }), diff --git a/packages/mcp-server/test/resources.test.ts b/packages/mcp-server/test/resources.test.ts index 094f618..d4a16c5 100644 --- a/packages/mcp-server/test/resources.test.ts +++ b/packages/mcp-server/test/resources.test.ts @@ -1,7 +1,12 @@ import fs from "node:fs"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import os from "node:os"; +import { + appendAction, + writeEvidenceReport, + type RunManifest, +} from "@pickforge/picklab-core"; import { connectLab, makeLabDirs, @@ -21,6 +26,36 @@ function first(contents: unknown): Record { return (contents as Array>)[0] as Record; } +async function seedEvidenceRun(projectDir: string): Promise { + const runDir = path.join(projectDir, ".picklab", "runs", RUN_ID); + const manifestPath = path.join(runDir, "manifest.json"); + const manifest = JSON.parse( + fs.readFileSync(manifestPath, "utf8"), + ) as RunManifest; + manifest.evidenceVersion = 1; + manifest.actionLog = "actions.jsonl"; + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + fs.writeFileSync(path.join(runDir, "actions.jsonl"), ""); + await appendAction(runDir, { + actionId: "second", + source: '', + tool: "desktop_type", + startedAt: "2026-06-09T12:00:04.000Z", + status: "error", + target: { label: "" }, + error: `Authorization: Bearer ${PLANTED_TOKEN}`, + }); + await appendAction(runDir, { + actionId: "first", + source: "mcp", + tool: "desktop_click", + startedAt: "2026-06-09T12:00:03.000Z", + status: "ok", + }); + await writeEvidenceReport(runDir, manifest); + return runDir; +} + let dirs: LabDirs; let lab: ConnectedLab; let sessionId: string; @@ -28,6 +63,7 @@ let sessionId: string; beforeEach(async () => { dirs = makeLabDirs(); writeSyntheticRun(dirs.projectDir, RUN_ID); + await seedEvidenceRun(dirs.projectDir); sessionId = writeDesktopSessionRecord(dirs.home, dirs.projectDir); lab = await connectLab({ projectDir: dirs.projectDir, @@ -41,11 +77,13 @@ afterEach(async () => { }); describe("resource listing", () => { - it("lists runs, manifests, screenshots, logs, and session statuses", async () => { + it("lists run evidence, artifacts, and session statuses", async () => { const { resources } = await lab.client.listResources(); const uris = resources.map((resource) => resource.uri); expect(uris).toContain("picklab://runs"); expect(uris).toContain(`picklab://runs/${RUN_ID}/manifest`); + expect(uris).toContain(`picklab://runs/${RUN_ID}/actions`); + expect(uris).toContain(`picklab://runs/${RUN_ID}/report`); expect(uris).toContain( `picklab://runs/${RUN_ID}/screenshots/screenshot.png`, ); @@ -61,6 +99,8 @@ describe("resource listing", () => { expect(templates).toEqual( expect.arrayContaining([ "picklab://runs/{runId}/manifest", + "picklab://runs/{runId}/actions", + "picklab://runs/{runId}/report", "picklab://runs/{runId}/screenshots/{name}", "picklab://runs/{runId}/logs/{name}", "picklab://sessions/{sessionId}/status", @@ -88,6 +128,83 @@ describe("resource reads", () => { expect(manifest.artifacts).toHaveLength(2); }); + it("reads deterministically ordered actions with secrets redacted", async () => { + const { contents } = await lab.client.readResource({ + uri: `picklab://runs/${RUN_ID}/actions`, + }); + expect(first(contents).mimeType).toBe("application/json"); + const text = first(contents).text as string; + const actions = JSON.parse(text) as Array>; + expect(actions.map((action) => action.actionId)).toEqual(["first", "second"]); + expect(text).toContain("[REDACTED]"); + expect(text).not.toContain(PLANTED_TOKEN); + }); + + it("reads the escaped static HTML report without planted secrets", async () => { + const { contents } = await lab.client.readResource({ + uri: `picklab://runs/${RUN_ID}/report`, + }); + expect(first(contents).mimeType).toBe("text/html"); + const html = first(contents).text as string; + expect(html).toContain("Content-Security-Policy"); + expect(html).toContain("</dd><script>alert(1)</script>"); + expect(html).not.toContain(" { + const legacyId = "20260609-110000-legacy"; + writeSyntheticRun(dirs.projectDir, legacyId); + + const { resources } = await lab.client.listResources(); + const uris = resources.map((resource) => resource.uri); + expect(uris).not.toContain(`picklab://runs/${legacyId}/actions`); + expect(uris).not.toContain(`picklab://runs/${legacyId}/report`); + await expect( + lab.client.readResource({ uri: `picklab://runs/${legacyId}/actions` }), + ).rejects.toThrow(/not found/i); + await expect( + lab.client.readResource({ uri: `picklab://runs/${legacyId}/report` }), + ).rejects.toThrow(/not found/i); + }); + + it("reports a missing evidence journal as not found", async () => { + const actionsPath = path.join( + dirs.projectDir, + ".picklab", + "runs", + RUN_ID, + "actions.jsonl", + ); + fs.rmSync(actionsPath); + + await expect( + lab.client.readResource({ uri: `picklab://runs/${RUN_ID}/actions` }), + ).rejects.toThrow(/not found/i); + const { resources } = await lab.client.listResources(); + expect(resources.map((resource) => resource.uri)).not.toContain( + `picklab://runs/${RUN_ID}/actions`, + ); + }); + + it("reports malformed evidence journals without returning partial data", async () => { + const actionsPath = path.join( + dirs.projectDir, + ".picklab", + "runs", + RUN_ID, + "actions.jsonl", + ); + fs.writeFileSync(actionsPath, '{"actionId":"ok"}\nnot-json\n'); + + const result = lab.client.readResource({ + uri: `picklab://runs/${RUN_ID}/actions`, + }); + await expect(result).rejects.toThrow(/Corrupt evidence journal/); + await expect(result).rejects.not.toThrow(dirs.projectDir); + }); + it("reads a screenshot as a base64 blob", async () => { const { contents } = await lab.client.readResource({ uri: `picklab://runs/${RUN_ID}/screenshots/screenshot.png`, @@ -164,6 +281,87 @@ describe("traversal protection", () => { }); describe("symlink protection", () => { + it.each([ + ["actions", "actions.jsonl"], + ["report", "report.html"], + ])("rejects a symlinked %s evidence file", async (resource, fileName) => { + const outside = path.join(dirs.root, `outside-${fileName}`); + fs.writeFileSync(outside, `token=${PLANTED_TOKEN}\n`); + const filePath = path.join( + dirs.projectDir, + ".picklab", + "runs", + RUN_ID, + fileName, + ); + fs.rmSync(filePath, { force: true }); + fs.symlinkSync(outside, filePath); + + await expect( + lab.client.readResource({ + uri: `picklab://runs/${RUN_ID}/${resource}`, + }), + ).rejects.toThrow(/not found/i); + + const { resources } = await lab.client.listResources(); + expect(resources.map((entry) => entry.uri)).not.toContain( + `picklab://runs/${RUN_ID}/${resource}`, + ); + }); + + it.each([ + ["actions", "actions.jsonl"], + ["report", "report.html"], + ])( + "rejects a parent-directory swap before opening %s", + async (resource, fileName) => { + const runDir = path.join( + dirs.projectDir, + ".picklab", + "runs", + RUN_ID, + ); + const backupRun = `${runDir}-backup`; + const outsideRun = path.join(dirs.root, `outside-${resource}-run`); + fs.mkdirSync(outsideRun, { recursive: true }); + fs.writeFileSync( + path.join(outsideRun, fileName), + resource === "actions" + ? `${JSON.stringify({ actionId: PLANTED_TOKEN })}\n` + : `${PLANTED_TOKEN}`, + ); + + const target = path.join(runDir, fileName); + const realOpen = fs.promises.open.bind(fs.promises); + let swapped = false; + const openSpy = vi + .spyOn(fs.promises, "open") + .mockImplementation(async (...args) => { + if (!swapped && path.resolve(String(args[0])) === target) { + swapped = true; + fs.renameSync(runDir, backupRun); + fs.symlinkSync(outsideRun, runDir); + } + return realOpen(...args); + }); + + try { + const result = lab.client.readResource({ + uri: `picklab://runs/${RUN_ID}/${resource}`, + }); + await expect(result).rejects.toThrow(/not found/i); + await expect(result).rejects.not.toThrow(PLANTED_TOKEN); + expect(swapped).toBe(true); + } finally { + openSpy.mockRestore(); + if (fs.existsSync(runDir) && fs.lstatSync(runDir).isSymbolicLink()) { + fs.unlinkSync(runDir); + } + if (fs.existsSync(backupRun)) fs.renameSync(backupRun, runDir); + } + }, + ); + it("rejects a screenshot symlink pointing outside the run dir", async () => { const secret = path.join(dirs.root, "outside-secret.png"); fs.writeFileSync(secret, Buffer.concat([PNG_MAGIC, Buffer.from([9])])); diff --git a/packages/mcp-server/test/tools.test.ts b/packages/mcp-server/test/tools.test.ts index b23f22c..c66697a 100644 --- a/packages/mcp-server/test/tools.test.ts +++ b/packages/mcp-server/test/tools.test.ts @@ -9,7 +9,9 @@ import { connectLab, makeLabDirs, parseToolJson, + PLANTED_TOKEN, removeLabDirs, + writeSyntheticRun, type ConnectedLab, type LabDirs, } from "./helpers.js"; @@ -217,6 +219,51 @@ describe("input validation", () => { }); }); +describe("artifact report", () => { + it("includes a deterministic redacted evidence timeline", async () => { + const runId = "20260609-120000-evidence"; + const { dir } = writeSyntheticRun(dirs.projectDir, runId); + const manifestPath = path.join(dir, "manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + manifest.evidenceVersion = 1; + manifest.actionLog = "actions.jsonl"; + fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + fs.writeFileSync( + path.join(dir, "actions.jsonl"), + [ + JSON.stringify({ + actionId: "second", + source: "mcp", + tool: "desktop_type", + startedAt: "2026-06-09T12:00:04.000Z", + status: "error", + error: `Authorization: Bearer ${PLANTED_TOKEN}`, + }), + JSON.stringify({ + actionId: "first", + source: "mcp", + tool: "desktop_click", + startedAt: "2026-06-09T12:00:03.000Z", + status: "ok", + }), + "", + ].join("\n"), + ); + + const result = await lab.client.callTool({ + name: "artifact_report", + arguments: { runId }, + }); + expect(result.isError).toBeFalsy(); + const report = parseToolJson(result).report as string; + expect(report.indexOf("Step 1 — mcp / desktop_click")).toBeLessThan( + report.indexOf("Step 2 — mcp / desktop_type"), + ); + expect(report).toContain("[REDACTED]"); + expect(report).not.toContain(PLANTED_TOKEN); + }); +}); + describe("empty lab", () => { it("reports no sessions", async () => { const result = await lab.client.callTool({