From 3d5c44db8ba5d5ac4d4f315e702f17b8eafd3988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elberte=20Pl=C3=ADnio?= Date: Mon, 13 Jul 2026 16:51:36 -0300 Subject: [PATCH] feat(browser): record DevTools evidence --- docs/releases/UNRELEASED.md | 19 + packages/browser/src/devtools-evidence.ts | 532 ++++++++++++++++++ packages/browser/src/devtools-mcp.ts | 84 ++- .../browser/test/devtools-evidence.test.ts | 491 ++++++++++++++++ 4 files changed, 1113 insertions(+), 13 deletions(-) create mode 100644 packages/browser/src/devtools-evidence.ts create mode 100644 packages/browser/test/devtools-evidence.test.ts diff --git a/docs/releases/UNRELEASED.md b/docs/releases/UNRELEASED.md index 349d820..2d33e69 100644 --- a/docs/releases/UNRELEASED.md +++ b/docs/releases/UNRELEASED.md @@ -131,6 +131,16 @@ then reset this file. existing one-shot screenshot run. Session destroy finalizes and releases the run, and dead-session reaping finalizes it as failed. Evidence failures are redacted, stderr-only, and never change the underlying tool result. +- The browser DevTools relay now correlates `tools/call` requests and responses + into the same session evidence run. It persists only allowlisted target + metadata, typed-value lengths/types, bounded sanitized errors, failed network + request metadata (method, origin/path, status/type/timing/error), and + warning/error console summaries; never request/response bodies, headers, + query strings, userinfo, raw arguments, evaluated code, or typed text. + Explicit inline PNG screenshot results are copied into the run and associated + with their action. Pending calls are closed on relay exit, diagnostic fan-out + is bounded, `evidence.enabled: false` is honored, and all evidence failures + remain redacted and relay-transparent. - Hosted CI now installs `x11vnc` alongside the other desktop test dependencies, and the desktop-linux integration suite asserts `x11vnc` is present when `CI=true` so VNC tests fail loudly instead of silently @@ -288,6 +298,15 @@ then reset this file. disabled/no-session behavior, evidence-write failure isolation, active-run reuse, explicit/shared session finalization, in-flight claim coordination, and dead-session reaping. Two independent changed-HEAD reviews are clean. +- DevTools evidence instrumentation: `bun run typecheck` and `bun run build` + pass; the full and coverage suites pass 76 files / 942 passed / 2 skipped + with all global thresholds met. Focused tests cover request/response + correlation, strict tool/target allowlists, typed/form privacy, URL stripping, + fail-closed JSON-RPC and tool-result errors, pinned-directory inline PNG + association and cap cleanup, failed-network and severity-only console + diagnostics, malformed/unmatched traffic, pending-call closure, disabled + configuration, evidence-write failure isolation, and bounded diagnostics. + Two independent changed-HEAD reviews are clean. ### Not tested yet diff --git a/packages/browser/src/devtools-evidence.ts b/packages/browser/src/devtools-evidence.ts new file mode 100644 index 0000000..5470557 --- /dev/null +++ b/packages/browser/src/devtools-evidence.ts @@ -0,0 +1,532 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { + appendAction, + beginEvidenceRun, + isEvidenceEnabled, + loadConfig, + sanitizeActionTarget, + sanitizeErrorText, + sanitizeNetworkFailure, + sanitizeTypedValue, + type EvidenceAction, + type AppendOutcome, + type EnvLike, + type RunHandle, +} from "@pickforge/picklab-core"; +import type { JsonRpcHook, JsonRpcMessage } from "./ndjson.js"; + +const MAX_PENDING_ACTIONS = 1_024; +const MAX_DIAGNOSTICS_PER_RESPONSE = 100; +const MAX_INLINE_SCREENSHOT_BYTES = 4 * 1024 * 1024; +const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const SUPPORTED_TOOL_NAMES: Readonly> = { + click: true, + click_at: true, + close_heapsnapshot: true, + drag: true, + close_page: true, + compare_heapsnapshots: true, + evaluate: true, + emulate: true, + evaluate_script: true, + execute_3p_developer_tool: true, + execute_webmcp_tool: true, + fill: true, + fill_form: true, + get_console_message: true, + get_heapsnapshot_class_nodes: true, + get_heapsnapshot_details: true, + get_heapsnapshot_dominators: true, + get_heapsnapshot_duplicate_strings: true, + get_heapsnapshot_edges: true, + get_heapsnapshot_retainers: true, + get_heapsnapshot_retaining_paths: true, + get_heapsnapshot_summary: true, + get_network_request: true, + get_tab_id: true, + handle_dialog: true, + hover: true, + install_extension: true, + lighthouse_audit: true, + list_3p_developer_tools: true, + list_console_messages: true, + list_extensions: true, + list_network_requests: true, + list_pages: true, + list_webmcp_tools: true, + navigate_page: true, + new_page: true, + performance_analyze_insight: true, + performance_start_trace: true, + performance_stop_trace: true, + press_key: true, + reload_extension: true, + resize_page: true, + screencast_start: true, + screencast_stop: true, + navigate: true, + select_page: true, + take_heapsnapshot: true, + take_screenshot: true, + take_snapshot: true, + trigger_extension_action: true, + type_text: true, + uninstall_extension: true, + upload_file: true, + wait_for: true, + screenshot: true, +}; +const NAVIGATION_TYPES: Readonly> = { + url: true, + back: true, + forward: true, + reload: true, +}; +const UID_PATTERN = /^\d+_\d+$/; +const TYPED_ARGUMENT_KEYS: Readonly> = { + fill: "value", + type_text: "text", + press_key: "key", + handle_dialog: "promptText", + evaluate_script: "function", +}; + +interface PendingAction { + actionId: string; + startedAt: Date; + tool: string; + target?: Record; +} + +export interface DevtoolsEvidenceRecorder { + beforeForward: JsonRpcHook; + afterResponse: JsonRpcHook; + flushPending(status?: EvidenceAction["status"]): Promise; +} + +export interface CreateDevtoolsEvidenceRecorderOptions { + projectDir: string; + sessionId: string; + env?: EnvLike; + reportFailure?: (detail: string) => void; + /** Injectable evidence cap for deterministic boundary tests. */ + maxBytes?: number; +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requestId(message: JsonRpcMessage): string | number | undefined { + return typeof message.id === "string" || typeof message.id === "number" + ? message.id + : undefined; +} + +function toolCall(message: JsonRpcMessage): { + id: string | number; + name: string; + args: Record; +} | undefined { + if (message.method !== "tools/call") return undefined; + const id = requestId(message); + if (id === undefined || !isObject(message.params)) return undefined; + const name = message.params.name; + if (typeof name !== "string" || !isObject(message.params.arguments)) { + return undefined; + } + return { id, name, args: message.params.arguments }; +} + +function persistedToolName(name: string): string { + return SUPPORTED_TOOL_NAMES[name] === true + ? `chrome_devtools/${name}` + : "chrome_devtools/unknown"; +} + +function typedMetadata( + name: string, + args: Record, +): Record | undefined { + const key = TYPED_ARGUMENT_KEYS[name]; + if (key !== undefined && typeof args[key] === "string") { + return { ...sanitizeTypedValue(args[key], "other") }; + } + if (name !== "fill_form" || !Array.isArray(args.elements)) return undefined; + let length = 0; + let fieldCount = 0; + for (const element of args.elements) { + if (!isObject(element) || typeof element.value !== "string") continue; + length += element.value.length; + fieldCount += 1; + } + return fieldCount === 0 + ? undefined + : { length, inputType: "other", fieldCount }; +} + +function actionTarget( + name: string, + args: Record, +): Record | undefined { + const raw: Record = {}; + if (typeof args.url === "string") raw.url = args.url; + if (typeof args.uid === "string" && UID_PATTERN.test(args.uid)) { + raw.selector = args.uid; + } + if (typeof args.x === "number") raw.x = args.x; + if (typeof args.y === "number") raw.y = args.y; + if ( + name === "navigate_page" && + typeof args.type === "string" && + NAVIGATION_TYPES[args.type] === true + ) { + raw.name = args.type; + } + const target: Record = { ...sanitizeActionTarget(raw) }; + const typed = typedMetadata(name, args); + if (typed !== undefined) Object.assign(target, typed); + return Object.keys(target).length === 0 ? undefined : target; +} + +function responseError(message: JsonRpcMessage): string | undefined { + return Object.prototype.hasOwnProperty.call(message, "error") || + (isObject(message.result) && message.result.isError === true) + ? "DevTools tool failed" + : undefined; +} + +function structuredContent(message: JsonRpcMessage): Record | undefined { + return isObject(message.result) && isObject(message.result.structuredContent) + ? message.result.structuredContent + : undefined; +} + +function diagnosticActions( + message: JsonRpcMessage, + sessionId: string, + startedAt: string, +): EvidenceAction[] { + const structured = structuredContent(message); + if (structured === undefined) return []; + const actions: EvidenceAction[] = []; + if (Array.isArray(structured.networkRequests)) { + for (const request of structured.networkRequests) { + if (actions.length >= MAX_DIAGNOSTICS_PER_RESPONSE) break; + if (!isObject(request)) continue; + const rawStatus = request.status; + const numericStatus = + typeof rawStatus === "number" + ? rawStatus + : typeof rawStatus === "string" && /^\d{3}$/.test(rawStatus) + ? Number(rawStatus) + : undefined; + const failedWithoutStatus = + typeof rawStatus === "string" && + numericStatus === undefined && + rawStatus !== "pending"; + const error = failedWithoutStatus + ? /^net::[A-Z0-9_]+$/.test(rawStatus) + ? rawStatus + : "Network request failed" + : undefined; + if ( + (numericStatus === undefined || numericStatus < 400) && + error === undefined + ) { + continue; + } + actions.push({ + actionId: crypto.randomUUID(), + source: "devtools", + tool: "network_failure", + sessionId, + startedAt, + status: "error", + target: { + ...sanitizeNetworkFailure({ + method: + typeof request.method === "string" ? request.method : undefined, + url: typeof request.url === "string" ? request.url : undefined, + status: numericStatus, + resourceType: + typeof request.resourceType === "string" + ? request.resourceType + : undefined, + durationMs: + typeof request.durationMs === "number" + ? request.durationMs + : undefined, + error, + }), + }, + }); + } + } + if (Array.isArray(structured.consoleMessages)) { + for (const message of structured.consoleMessages) { + if (actions.length >= MAX_DIAGNOSTICS_PER_RESPONSE) break; + if (!isObject(message) || (message.type !== "error" && message.type !== "warning")) { + continue; + } + actions.push({ + actionId: crypto.randomUUID(), + source: "devtools", + tool: "console_message", + sessionId, + startedAt, + status: "error", + target: { role: message.type }, + error: `Console ${message.type}`, + }); + } + } + return actions; +} + +interface RunIdentity { + dev: number; + ino: number; +} + +function descriptorRoot(): string { + if (process.platform === "linux") return "/proc/self/fd"; + if (process.platform === "darwin") return "/dev/fd"; + throw new Error( + `Safe inline screenshot writes are unsupported on ${process.platform}`, + ); +} + +async function withPinnedScreenshotDirectory( + runDir: string, + runIdentity: RunIdentity, + operation: (directoryPath: string) => Promise, +): Promise { + const fdRoot = descriptorRoot(); + const directoryFlags = + fs.constants.O_RDONLY | + fs.constants.O_DIRECTORY | + fs.constants.O_NOFOLLOW; + const runHandle = await fs.promises.open(runDir, directoryFlags); + try { + const stat = await runHandle.stat(); + if ( + !stat.isDirectory() || + stat.dev !== runIdentity.dev || + stat.ino !== runIdentity.ino + ) { + throw new Error("Evidence run directory changed before screenshot access"); + } + const screenshotsHandle = await fs.promises.open( + path.join(fdRoot, String(runHandle.fd), "screenshots"), + directoryFlags, + ); + try { + return await operation( + path.join(fdRoot, String(screenshotsHandle.fd)), + ); + } finally { + await screenshotsHandle.close(); + } + } finally { + await runHandle.close(); + } +} + +async function writePinnedScreenshot( + runDir: string, + runIdentity: RunIdentity, + filename: string, + bytes: Buffer, +): Promise { + await withPinnedScreenshotDirectory( + runDir, + runIdentity, + async (directoryPath) => { + await fs.promises.writeFile(path.join(directoryPath, filename), bytes, { + flag: "wx", + mode: 0o600, + }); + }, + ); +} + +async function removePinnedScreenshot( + runDir: string, + runIdentity: RunIdentity, + filename: string, +): Promise { + await withPinnedScreenshotDirectory( + runDir, + runIdentity, + async (directoryPath) => { + await fs.promises.rm(path.join(directoryPath, filename), { force: true }); + }, + ); +} + +async function captureInlinePng( + message: JsonRpcMessage, + run: RunHandle, + runIdentity: RunIdentity, + actionId: string, +): Promise { + if (!isObject(message.result) || !Array.isArray(message.result.content)) { + return undefined; + } + const image = message.result.content.find( + (entry) => + isObject(entry) && + entry.type === "image" && + entry.mimeType === "image/png" && + typeof entry.data === "string", + ); + if (!isObject(image) || typeof image.data !== "string") return undefined; + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(image.data)) return undefined; + const estimatedBytes = Math.floor((image.data.length * 3) / 4); + if (estimatedBytes > MAX_INLINE_SCREENSHOT_BYTES) return undefined; + const bytes = Buffer.from(image.data, "base64"); + if (bytes.length < PNG_MAGIC.length || !bytes.subarray(0, PNG_MAGIC.length).equals(PNG_MAGIC)) { + return undefined; + } + const filename = `devtools-${actionId}.png`; + await writePinnedScreenshot(run.dir, runIdentity, filename, bytes); + return path.join("screenshots", filename); +} + +export async function createDevtoolsEvidenceRecorder( + opts: CreateDevtoolsEvidenceRecorderOptions, +): Promise { + const config = await loadConfig(opts.projectDir, opts.env); + if (!isEvidenceEnabled(config)) return undefined; + const { run } = await beginEvidenceRun(opts.projectDir, opts.sessionId, { + slug: "computer-use", + }); + const runStat = await fs.promises.lstat(run.dir); + if (!runStat.isDirectory() || runStat.isSymbolicLink()) { + throw new Error("Evidence run directory is not a real directory"); + } + const runIdentity: RunIdentity = { dev: runStat.dev, ino: runStat.ino }; + const pending = new Map(); + const report = (error: unknown): void => { + const detail = sanitizeErrorText( + error instanceof Error ? error.message : String(error), + ); + try { + opts.reportFailure?.(detail); + } catch { + // Evidence diagnostics must never break the relay. + } + }; + const append = async ( + action: EvidenceAction, + ): Promise => { + try { + const result = await appendAction(run.dir, action, { + maxBytes: opts.maxBytes, + }); + return result.outcome; + } catch (error) { + report(error); + return undefined; + } + }; + + return { + beforeForward: async (message) => { + try { + const call = toolCall(message); + if (call === undefined || pending.size >= MAX_PENDING_ACTIONS) return; + pending.set(call.id, { + actionId: crypto.randomUUID(), + startedAt: new Date(), + tool: persistedToolName(call.name), + target: actionTarget(call.name, call.args), + }); + } catch (error) { + report(error); + } + }, + afterResponse: async (message) => { + try { + const id = requestId(message); + if (id === undefined) return; + const action = pending.get(id); + if (action === undefined) return; + pending.delete(id); + const error = responseError(message); + const artifacts: string[] = []; + if ( + (action.tool === "chrome_devtools/take_screenshot" || + action.tool === "chrome_devtools/screenshot") && + error === undefined + ) { + try { + const screenshot = await captureInlinePng( + message, + run, + runIdentity, + action.actionId, + ); + if (screenshot !== undefined) artifacts.push(screenshot); + } catch (captureError) { + report(captureError); + } + } + const record: EvidenceAction = { + actionId: action.actionId, + source: "devtools", + tool: action.tool, + sessionId: opts.sessionId, + startedAt: action.startedAt.toISOString(), + durationMs: Date.now() - action.startedAt.getTime(), + status: error === undefined ? "ok" : "error", + }; + if (action.target !== undefined) record.target = action.target; + if (artifacts.length > 0) record.artifacts = artifacts; + if (error !== undefined) record.error = error; + const outcome = await append(record); + if ( + artifacts.length > 0 && + (outcome === "capped" || outcome === undefined) + ) { + for (const artifact of artifacts) { + await removePinnedScreenshot( + run.dir, + runIdentity, + path.basename(artifact), + ).catch(report); + } + } + for (const diagnostic of diagnosticActions( + message, + opts.sessionId, + new Date().toISOString(), + )) { + await append(diagnostic); + } + } catch (error) { + report(error); + } + }, + flushPending: async (status = "cancelled") => { + const unfinished = [...pending.values()]; + pending.clear(); + for (const action of unfinished) { + await append({ + actionId: action.actionId, + source: "devtools", + tool: action.tool, + sessionId: opts.sessionId, + startedAt: action.startedAt.toISOString(), + durationMs: Date.now() - action.startedAt.getTime(), + status, + ...(action.target === undefined ? {} : { target: action.target }), + error: "DevTools relay ended before the tool returned", + }); + } + }, + }; +} diff --git a/packages/browser/src/devtools-mcp.ts b/packages/browser/src/devtools-mcp.ts index f736f82..8b19670 100644 --- a/packages/browser/src/devtools-mcp.ts +++ b/packages/browser/src/devtools-mcp.ts @@ -11,10 +11,15 @@ import type { Readable, Writable } from "node:stream"; import { listSessions, redactSecrets, + sanitizeErrorText, type EnvLike, type SessionRecord, } from "@pickforge/picklab-core"; import { createDeferred } from "./deferred.js"; +import { + createDevtoolsEvidenceRecorder, + type DevtoolsEvidenceRecorder, +} from "./devtools-evidence.js"; import { getBrowserSessionStatus, type BrowserSessionStatus } from "./session.js"; import { JsonRpcProtocolError, @@ -177,6 +182,24 @@ export interface RelayHooks { afterResponse?: JsonRpcHook; } +function composeEvidenceHooks( + hooks: RelayHooks | undefined, + evidence: DevtoolsEvidenceRecorder | undefined, +): RelayHooks | undefined { + if (evidence === undefined) return hooks; + return { + beforeForward: async (message) => { + const transformed = await hooks?.beforeForward?.(message); + await evidence.beforeForward(transformed ?? message); + return transformed; + }, + afterResponse: async (message) => { + await evidence.afterResponse(message); + return hooks?.afterResponse?.(message); + }, + }; +} + export interface RelayExit { code: number | null; signal: NodeJS.Signals | null; @@ -485,6 +508,19 @@ export interface RunProjectDevtoolsMcpOptions { maxDiagnosticLineBytes?: number; } +function reportProjectEvidenceFailure( + destination: Writable, + error: unknown, +): void { + const detail = sanitizeErrorText( + error instanceof Error ? error.message : String(error), + ); + void writeWithBackpressure( + destination, + Buffer.from(`[picklab evidence] chrome-devtools: ${detail}\n`), + ).catch(() => {}); +} + export async function runProjectDevtoolsMcp( opts: RunProjectDevtoolsMcpOptions, ): Promise { @@ -492,17 +528,39 @@ export async function runProjectDevtoolsMcp( resolveLiveBrowserSession({ projectDir: opts.projectDir, env: opts.env }), resolveDevtoolsMcpExecutable(), ]); - return runDevtoolsMcpRelay({ - session, - executable, - input: opts.input, - output: opts.output, - diagnostics: opts.diagnostics, - env: opts.env, - cwd: path.resolve(opts.projectDir), - hooks: opts.hooks, - shutdownTimeoutMs: opts.shutdownTimeoutMs, - maxRecordBytes: opts.maxRecordBytes, - maxDiagnosticLineBytes: opts.maxDiagnosticLineBytes, - }); + const diagnostics = opts.diagnostics ?? process.stderr; + let evidenceFailureReported = false; + const reportEvidenceFailure = (error: unknown): void => { + if (evidenceFailureReported) return; + evidenceFailureReported = true; + reportProjectEvidenceFailure(diagnostics, error); + }; + let evidence: DevtoolsEvidenceRecorder | undefined; + try { + evidence = await createDevtoolsEvidenceRecorder({ + projectDir: opts.projectDir, + sessionId: session.record.id, + env: opts.env, + reportFailure: reportEvidenceFailure, + }); + } catch (error) { + reportEvidenceFailure(error); + } + try { + return await runDevtoolsMcpRelay({ + session, + executable, + input: opts.input, + output: opts.output, + diagnostics, + env: opts.env, + cwd: path.resolve(opts.projectDir), + hooks: composeEvidenceHooks(opts.hooks, evidence), + shutdownTimeoutMs: opts.shutdownTimeoutMs, + maxRecordBytes: opts.maxRecordBytes, + maxDiagnosticLineBytes: opts.maxDiagnosticLineBytes, + }); + } finally { + await evidence?.flushPending(); + } } diff --git a/packages/browser/test/devtools-evidence.test.ts b/packages/browser/test/devtools-evidence.test.ts new file mode 100644 index 0000000..fddb0f3 --- /dev/null +++ b/packages/browser/test/devtools-evidence.test.ts @@ -0,0 +1,491 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + listRuns, + readActions, + saveProjectConfig, + type EvidenceAction, +} from "@pickforge/picklab-core"; +import { createDevtoolsEvidenceRecorder } from "../src/devtools-evidence.js"; +import type { JsonRpcMessage } from "../src/ndjson.js"; + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const SECRET = "picklab-devtools-secret"; +let projectDir: string; + +beforeEach(async () => { + projectDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), "picklab-devtools-evidence-"), + ); +}); + +afterEach(async () => { + await fs.promises.rm(projectDir, { recursive: true, force: true }); +}); + +function call( + id: string | number, + name: string, + args: Record, +): JsonRpcMessage { + return { + jsonrpc: "2.0", + id, + method: "tools/call", + params: { name, arguments: args }, + }; +} + +function response( + id: string | number, + result: Record, +): JsonRpcMessage { + return { jsonrpc: "2.0", id, result }; +} + +async function recorder(reportFailure?: (detail: string) => void) { + const created = await createDevtoolsEvidenceRecorder({ + projectDir, + sessionId: "browser-evidence", + reportFailure, + }); + expect(created).toBeDefined(); + return created!; +} + +async function actions(): Promise { + const [manifest] = await listRuns(projectDir); + expect(manifest).toBeDefined(); + const records = await readActions( + path.join(projectDir, ".picklab", "runs", manifest!.runId), + ); + return records.filter( + (record): record is EvidenceAction => "tool" in record, + ); +} + +describe("DevTools evidence recorder", () => { + it("correlates tool responses without persisting typed values or URL queries", async () => { + const evidence = await recorder(); + await evidence.beforeForward( + call(1, "fill", { uid: "1_1", value: SECRET }), + ); + await evidence.afterResponse(response(1, { content: [] })); + await evidence.beforeForward( + call(2, "navigate_page", { + type: "url", + url: `https://user:${SECRET}@example.com/path?token=${SECRET}#hash`, + }), + ); + await evidence.afterResponse(response(2, { content: [] })); + + const records = await actions(); + expect(records).toMatchObject([ + { + source: "devtools", + tool: "chrome_devtools/fill", + status: "ok", + target: { + selector: "1_1", + length: SECRET.length, + inputType: "other", + }, + }, + { + source: "devtools", + tool: "chrome_devtools/navigate_page", + status: "ok", + target: { name: "url", url: "https://example.com/path" }, + }, + ]); + expect(JSON.stringify(records)).not.toContain(SECRET); + }); + + it("records redacted JSON-RPC and tool-result failures", async () => { + const evidence = await recorder(); + await evidence.beforeForward(call(1, "click", { uid: "button" })); + await evidence.afterResponse({ + jsonrpc: "2.0", + id: 1, + error: { code: -32_000, message: `token=${SECRET}` }, + }); + await evidence.beforeForward(call(2, "hover", { uid: "card" })); + await evidence.afterResponse( + response(2, { + isError: true, + content: [{ type: "text", text: `Authorization: Bearer ${SECRET}` }], + }), + ); + + const records = await actions(); + expect(records).toMatchObject([ + { tool: "chrome_devtools/click", status: "error", error: "DevTools tool failed" }, + { tool: "chrome_devtools/hover", status: "error" }, + ]); + expect(JSON.stringify(records)).not.toContain(SECRET); + }); + + it("associates only explicit inline PNG screenshots", async () => { + const evidence = await recorder(); + await evidence.beforeForward(call(1, "take_screenshot", {})); + await evidence.afterResponse( + response(1, { + content: [ + { type: "image", mimeType: "image/png", data: PNG.toString("base64") }, + ], + }), + ); + + const [record] = await actions(); + expect(record?.artifacts).toHaveLength(1); + const [relative] = record!.artifacts!; + expect(relative).toMatch(/^screenshots\/devtools-.+\.png$/); + expect(fs.readFileSync(path.join(projectDir, ".picklab", "runs", (await listRuns(projectDir))[0]!.runId, relative!))).toEqual(PNG); + }); + + it("records only sanitized failed network and relevant console diagnostics", async () => { + const evidence = await recorder(); + await evidence.beforeForward(call(1, "list_network_requests", {})); + await evidence.afterResponse( + response(1, { + content: [], + structuredContent: { + networkRequests: [ + { method: "GET", url: "https://example.com/ok?token=secret", status: "200" }, + { + method: "POST", + url: `https://example.com/fail?token=${SECRET}`, + status: "503", + requestHeaders: { authorization: SECRET }, + responseBody: SECRET, + }, + { method: "GET", url: "https://example.com/offline", status: `net::${SECRET}` }, + { + method: "GET", + url: "https://example.com/missing?private=value", + status: 404, + resourceType: "fetch", + durationMs: 12.4, + }, + { + method: "GET", + url: "https://example.com/dns", + status: "net::ERR_NAME_NOT_RESOLVED", + }, + ], + consoleMessages: [ + { type: "log", text: SECRET }, + { type: "warning", text: `token=${SECRET}` }, + { type: "error", text: `Authorization: Bearer ${SECRET}` }, + ], + }, + }), + ); + + const records = await actions(); + expect(records).toHaveLength(7); + expect(records.slice(1)).toMatchObject([ + { + tool: "network_failure", + target: { method: "POST", url: "https://example.com/fail", status: 503 }, + }, + { + tool: "network_failure", + target: { method: "GET", url: "https://example.com/offline" }, + }, + { + tool: "network_failure", + target: { + method: "GET", + url: "https://example.com/missing", + status: 404, + resourceType: "fetch", + durationMs: 12, + }, + }, + { + tool: "network_failure", + target: { + method: "GET", + url: "https://example.com/dns", + error: "net::ERR_NAME_NOT_RESOLVED", + }, + }, + { + tool: "console_message", + target: { role: "warning" }, + error: "Console warning", + }, + { + tool: "console_message", + target: { role: "error" }, + error: "Console error", + }, + ]); + const serialized = JSON.stringify(records); + expect(serialized).not.toContain(SECRET); + expect(serialized).not.toContain("requestHeaders"); + expect(serialized).not.toContain("responseBody"); + }); + + it("flushes unanswered calls without persisting raw arguments", async () => { + const evidence = await recorder(); + await evidence.beforeForward(call(7, "press_key", { key: SECRET })); + + await evidence.flushPending("timeout"); + + const [record] = await actions(); + expect(record).toMatchObject({ + tool: "chrome_devtools/press_key", + status: "timeout", + target: { length: SECRET.length, inputType: "other" }, + }); + expect(JSON.stringify(record)).not.toContain(SECRET); + }); + + + it("handles aggregate typing, coordinates, and unsafe tool names", async () => { + const evidence = await recorder(); + await evidence.beforeForward( + call(1, "fill_form", { + elements: [ + { uid: "a", value: SECRET }, + { uid: "b", value: "two" }, + null, + { uid: "ignored", value: 42 }, + ], + }), + ); + await evidence.afterResponse(response(1, { content: [] })); + await evidence.beforeForward(call(2, `bad\n${SECRET}`, { x: 10, y: 20 })); + await evidence.afterResponse(response(2, { content: [] })); + await evidence.beforeForward(call(3, "click", { uid: SECRET })); + await evidence.afterResponse(response(3, { content: [] })); + await evidence.beforeForward( + call(4, "navigate_page", { + type: SECRET, + url: `https://example.com/safe?secret=${SECRET}`, + }), + ); + await evidence.afterResponse(response(4, { content: [] })); + await evidence.beforeForward(call(5, SECRET, {})); + await evidence.afterResponse(response(5, { content: [] })); + + expect(await actions()).toMatchObject([ + { + tool: "chrome_devtools/fill_form", + target: { + length: SECRET.length + 3, + inputType: "other", + fieldCount: 2, + }, + }, + { + tool: "chrome_devtools/unknown", + target: { x: 10, y: 20 }, + }, + { + tool: "chrome_devtools/click", + }, + { + tool: "chrome_devtools/navigate_page", + target: { url: "https://example.com/safe" }, + }, + { + tool: "chrome_devtools/unknown", + }, + ]); + expect(JSON.stringify(await actions())).not.toContain(SECRET); + }); + + it("ignores malformed and unmatched JSON-RPC traffic", async () => { + const evidence = await recorder(); + await evidence.beforeForward({ jsonrpc: "2.0", method: "tools/list" }); + await evidence.beforeForward({ + jsonrpc: "2.0", + method: "tools/call", + params: { name: "click", arguments: {} }, + }); + await evidence.beforeForward({ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: [], + }); + await evidence.beforeForward({ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { name: 42, arguments: SECRET }, + }); + await evidence.afterResponse(response(99, { content: [] })); + await evidence.afterResponse({ + jsonrpc: "2.0", + id: null, + result: { content: [] }, + }); + + expect(await actions()).toEqual([]); + }); + + it("fails closed on malformed JSON-RPC error members", async () => { + const evidence = await recorder(); + await evidence.beforeForward(call(1, "click", {})); + await evidence.afterResponse({ + jsonrpc: "2.0", + id: 1, + error: { code: -32_000 }, + }); + await evidence.beforeForward(call(2, "hover", {})); + await evidence.afterResponse({ + jsonrpc: "2.0", + id: 2, + error: SECRET, + }); + + expect(await actions()).toMatchObject([ + { status: "error", error: "DevTools tool failed" }, + { status: "error", error: "DevTools tool failed" }, + ]); + expect(JSON.stringify(await actions())).not.toContain(SECRET); + }); + + it("omits invalid inline screenshot payloads but records their calls", async () => { + const evidence = await recorder(); + const payloads = [ + { type: "text", text: "not an image" }, + { type: "image", mimeType: "image/jpeg", data: PNG.toString("base64") }, + { type: "image", mimeType: "image/png", data: "not base64!" }, + { + type: "image", + mimeType: "image/png", + data: Buffer.from("not a png").toString("base64"), + }, + ]; + for (const [index, payload] of payloads.entries()) { + await evidence.beforeForward(call(index, "take_screenshot", {})); + await evidence.afterResponse( + response(index, { content: [payload] }), + ); + } + + const records = await actions(); + expect(records).toHaveLength(payloads.length); + expect(records.every((record) => record.artifacts === undefined)).toBe(true); + }); + + it("rejects a swapped screenshot directory without writing outside the run", async () => { + const failures: string[] = []; + const evidence = await recorder((detail) => failures.push(detail)); + const [manifest] = await listRuns(projectDir); + const runDir = path.join( + projectDir, + ".picklab", + "runs", + manifest!.runId, + ); + const outside = path.join(projectDir, "outside"); + await fs.promises.mkdir(outside); + await fs.promises.rm(path.join(runDir, "screenshots"), { + recursive: true, + }); + await fs.promises.symlink(outside, path.join(runDir, "screenshots")); + + await evidence.beforeForward(call(1, "take_screenshot", {})); + await evidence.afterResponse( + response(1, { + content: [ + { type: "image", mimeType: "image/png", data: PNG.toString("base64") }, + ], + }), + ); + + expect(await fs.promises.readdir(outside)).toEqual([]); + expect((await actions())[0]?.artifacts).toBeUndefined(); + expect(failures).toHaveLength(1); + }); + + it("removes inline screenshots rejected by the evidence cap", async () => { + const evidence = await createDevtoolsEvidenceRecorder({ + projectDir, + sessionId: "browser-capped", + maxBytes: 1, + }); + expect(evidence).toBeDefined(); + await evidence!.beforeForward(call(1, "take_screenshot", {})); + await evidence!.afterResponse( + response(1, { + content: [ + { type: "image", mimeType: "image/png", data: PNG.toString("base64") }, + ], + }), + ); + + const [manifest] = await listRuns(projectDir); + const screenshots = await fs.promises.readdir( + path.join(projectDir, ".picklab", "runs", manifest!.runId, "screenshots"), + ); + expect(screenshots).toEqual([]); + }); + + it("keeps inline screenshots whose action crosses the evidence cap", async () => { + const evidence = await createDevtoolsEvidenceRecorder({ + projectDir, + sessionId: "browser-truncated", + maxBytes: PNG.length + 50, + }); + expect(evidence).toBeDefined(); + await evidence!.beforeForward(call(1, "take_screenshot", {})); + await evidence!.afterResponse( + response(1, { + content: [ + { type: "image", mimeType: "image/png", data: PNG.toString("base64") }, + ], + }), + ); + + const [record] = await actions(); + expect(record?.artifacts).toHaveLength(1); + const [manifest] = await listRuns(projectDir); + expect( + fs.existsSync( + path.join( + projectDir, + ".picklab", + "runs", + manifest!.runId, + record!.artifacts![0]!, + ), + ), + ).toBe(true); + }); + it("is disabled by project configuration", async () => { + await saveProjectConfig(projectDir, { evidence: { enabled: false } }); + + expect( + await createDevtoolsEvidenceRecorder({ + projectDir, + sessionId: "browser-disabled", + }), + ).toBeUndefined(); + expect(await listRuns(projectDir)).toEqual([]); + }); + + it("reports evidence-write failures without rejecting relay hooks", async () => { + const failures: string[] = []; + const evidence = await recorder((detail) => failures.push(detail)); + await evidence.beforeForward(call(1, "click", { uid: "button" })); + const [manifest] = await listRuns(projectDir); + await fs.promises.rm( + path.join(projectDir, ".picklab", "runs", manifest!.runId), + { recursive: true, force: true }, + ); + + await expect( + evidence.afterResponse(response(1, { content: [] })), + ).resolves.toBeUndefined(); + expect(failures).toHaveLength(1); + }); +});