From 0a9e6033dd4e2caac5a0467f9d11331caac0418d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 14:44:29 +0000 Subject: [PATCH 1/2] fix(compliance): use truthiness form in evaluator_auth skip_if guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adcp-client runner mishandles boolean false in != true comparisons: context.supports_evaluator evaluates to false but false != true is evaluated as false instead of true, causing evaluator phases to run against agents that do not advertise evaluator support. Replace all six skip_if guards from "context.supports_evaluator != true" to "!context.supports_evaluator" — semantically equivalent for booleans and consistent with the existing !context.build_capability_id / !context.creative_feature_id style already in the compound guards. The underlying runner bug is tracked separately in adcp-client. Closes #5844 --- .../fix-evaluator-auth-skip-if-boolean-guard.md | 9 +++++++++ .../protocols/creative/scenarios/evaluator_auth.yaml | 12 ++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 .changeset/fix-evaluator-auth-skip-if-boolean-guard.md diff --git a/.changeset/fix-evaluator-auth-skip-if-boolean-guard.md b/.changeset/fix-evaluator-auth-skip-if-boolean-guard.md new file mode 100644 index 0000000000..64d967099c --- /dev/null +++ b/.changeset/fix-evaluator-auth-skip-if-boolean-guard.md @@ -0,0 +1,9 @@ +--- +"adcontextprotocol": patch +--- + +Fix `evaluator_auth` storyboard: use truthiness form `!context.supports_evaluator` in all `skip_if` guards. + +The storyboard used `context.supports_evaluator != true` (strict typed equality) to gate evaluator-specific phases. When a creative agent returns `creative.supports_evaluator: false`, the runner captures the boolean `false` into `context.supports_evaluator` but mishandles the `!= true` strict comparison — evaluating it as `false` instead of `true` — so the guarded phase executes instead of being skipped. The underlying bug is in the adcp-client runner's `skip_if` evaluator (tracked separately); this change applies a defensive storyboard workaround that is semantically equivalent for boolean values and avoids the typed equality path entirely. + +Changed all six `skip_if` guards from `"context.supports_evaluator != true ..."` to `"!context.supports_evaluator ..."`, matching the truthiness style already used for `context.build_capability_id` and `context.creative_feature_id` in the same expressions. The storyboard was correctly authored per protocol semantics; the change is purely defensive against the runner bug during the 3.1 pre-release window. diff --git a/static/compliance/source/protocols/creative/scenarios/evaluator_auth.yaml b/static/compliance/source/protocols/creative/scenarios/evaluator_auth.yaml index adf4ee4dae..ef031f2efe 100644 --- a/static/compliance/source/protocols/creative/scenarios/evaluator_auth.yaml +++ b/static/compliance/source/protocols/creative/scenarios/evaluator_auth.yaml @@ -111,7 +111,7 @@ phases: - id: evaluator_capability_contract title: "Confirm experimental feature declaration" optional: true - skip_if: "context.supports_evaluator != true" + skip_if: "!context.supports_evaluator" narrative: | Because evaluator support is experimental, agents that set `creative.supports_evaluator` must also list `creative.evaluator` in @@ -162,7 +162,7 @@ phases: - id: reject_off_list_evaluator title: "Reject an off-list evaluator URL" optional: true - skip_if: "context.supports_evaluator != true || !context.build_capability_id || !context.creative_feature_id" + skip_if: "!context.supports_evaluator || !context.build_capability_id || !context.creative_feature_id" narrative: | The buyer supplies a well-formed external evaluator URL that is not in the seller's accepted verifier allowlist. The creative agent must reject it @@ -219,7 +219,7 @@ phases: - id: reject_off_list_feature_agent title: "Reject an off-list feature_agent URL" optional: true - skip_if: "context.supports_evaluator != true || !context.build_capability_id || !context.creative_feature_id" + skip_if: "!context.supports_evaluator || !context.build_capability_id || !context.creative_feature_id" narrative: | The buyer supplies a well-formed `feature_agent.agent_url` alongside an account-arranged evaluator id. The nested feature agent uses the same @@ -281,7 +281,7 @@ phases: - id: payload_credentials_advisory title: "Keep evaluator credentials out of the payload" optional: true - skip_if: "context.supports_evaluator != true || !context.build_capability_id || !context.creative_feature_id" + skip_if: "!context.supports_evaluator || !context.build_capability_id || !context.creative_feature_id" narrative: | The buyer attempts to pass an evaluator API key inside the evaluator object. The payload is schema-valid but non-conforming: evaluator @@ -356,7 +356,7 @@ phases: - id: accepted_evaluator_invocation title: "Invoke an accepted evaluator" optional: true - skip_if: "context.supports_evaluator != true || !context.build_capability_id || !context.creative_feature_id" + skip_if: "!context.supports_evaluator || !context.build_capability_id || !context.creative_feature_id" narrative: | The buyer supplies an accepted external evaluator URL. The creative agent builds variants, calls the evaluator through the `get_creative_features` @@ -433,7 +433,7 @@ phases: - id: accepted_evaluator_unavailable_degrades title: "Degrade when an accepted evaluator is unavailable" optional: true - skip_if: "context.supports_evaluator != true || !context.build_capability_id || !context.creative_feature_id" + skip_if: "!context.supports_evaluator || !context.build_capability_id || !context.creative_feature_id" narrative: | An accepted evaluator can be unreachable or can reject the producing agent's transport authentication. The buyer-visible build should not fail From b4c9ef0501f2ef5c8c785904dcca989f1fc0edd2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 14:47:47 +0000 Subject: [PATCH 2/2] build: add compiled output for storyboard-runner-options Generated by tsc --project server/tsconfig.json; outDir is dist/, dist/compliance/ is explicitly un-ignored in .gitignore. File was added to server/src/compliance/ without its initial compiled output. --- .../compliance/storyboard-runner-options.d.ts | 42 +++++++++++ .../storyboard-runner-options.d.ts.map | 1 + dist/compliance/storyboard-runner-options.js | 69 +++++++++++++++++++ .../storyboard-runner-options.js.map | 1 + 4 files changed, 113 insertions(+) create mode 100644 dist/compliance/storyboard-runner-options.d.ts create mode 100644 dist/compliance/storyboard-runner-options.d.ts.map create mode 100644 dist/compliance/storyboard-runner-options.js create mode 100644 dist/compliance/storyboard-runner-options.js.map diff --git a/dist/compliance/storyboard-runner-options.d.ts b/dist/compliance/storyboard-runner-options.d.ts new file mode 100644 index 0000000000..b472fcdc42 --- /dev/null +++ b/dist/compliance/storyboard-runner-options.d.ts @@ -0,0 +1,42 @@ +import type { StoryboardRunOptions } from '@adcp/sdk/testing'; +export interface LoadedTestKit { + brand?: { + house?: { + domain?: string; + }; + brand_id?: string; + }; + auth?: { + api_key?: string; + basic?: { + username?: string; + password?: string; + credentials?: string; + }; + probe_task?: string; + }; +} +/** + * Per-tenant probe-task override for security_baseline's auth probes. + * + * Most shared test-kits declare `auth.probe_task: list_creatives`, but cached + * prerelease kits can lag the allowlist. Sales/creative explicitly pin the + * allowlisted protected read they serve. /signals and /governance serve + * different SDK-allowlisted protected reads. + */ +export declare const PROBE_TASK_BY_TENANT: Record; +/** + * Thread the test-kit's auth material through to the storyboard runner so + * kit-gated auth phases execute instead of being skipped by `skip_if`. + */ +export declare function testKitOptionsFromKit(kit: LoadedTestKit | undefined, tenantPath?: any): StoryboardRunOptions['test_kit'] | undefined; +/** + * Pick run-scoped transport auth for the manual storyboard runner. + * + * `security_baseline` positive credential probes use normal initialized + * transport calls, so a single run can only prove one static credential type. + * Dual-credential kits must be split into per-mechanism runs before they can + * be graded safely here. + */ +export declare function authForStoryboard(storyboardId: string, kit: LoadedTestKit | undefined, defaultBearerToken: string): StoryboardRunOptions['auth']; +//# sourceMappingURL=storyboard-runner-options.d.ts.map \ No newline at end of file diff --git a/dist/compliance/storyboard-runner-options.d.ts.map b/dist/compliance/storyboard-runner-options.d.ts.map new file mode 100644 index 0000000000..1c38ab5201 --- /dev/null +++ b/dist/compliance/storyboard-runner-options.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"storyboard-runner-options.d.ts","sourceRoot":"","sources":["../../server/src/compliance/storyboard-runner-options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE;YAAE,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D,IAAI,CAAC,EAAE;QACL,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,WAAW,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACvE,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAKvD,CAAC;AAEF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,GAAG,EAAE,aAAa,GAAG,SAAS,EAC9B,UAAU,MAA0B,GACnC,oBAAoB,CAAC,UAAU,CAAC,GAAG,SAAS,CAc9C;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,GAAG,EAAE,aAAa,GAAG,SAAS,EAC9B,kBAAkB,EAAE,MAAM,GACzB,oBAAoB,CAAC,MAAM,CAAC,CA8B9B"} \ No newline at end of file diff --git a/dist/compliance/storyboard-runner-options.js b/dist/compliance/storyboard-runner-options.js new file mode 100644 index 0000000000..c346296ea5 --- /dev/null +++ b/dist/compliance/storyboard-runner-options.js @@ -0,0 +1,69 @@ +/** + * Per-tenant probe-task override for security_baseline's auth probes. + * + * Most shared test-kits declare `auth.probe_task: list_creatives`, but cached + * prerelease kits can lag the allowlist. Sales/creative explicitly pin the + * allowlisted protected read they serve. /signals and /governance serve + * different SDK-allowlisted protected reads. + */ +export const PROBE_TASK_BY_TENANT = { + sales: 'list_creatives', + creative: 'list_creatives', + signals: 'get_signals', + governance: 'list_content_standards', +}; +/** + * Thread the test-kit's auth material through to the storyboard runner so + * kit-gated auth phases execute instead of being skipped by `skip_if`. + */ +export function testKitOptionsFromKit(kit, tenantPath = process.env.TENANT_PATH) { + const auth = kit?.auth; + if (!auth?.api_key && !auth?.basic && !auth?.probe_task) + return undefined; + if (!auth.probe_task) { + throw new Error('test kit declares auth credentials without auth.probe_task — required by runner'); + } + const probeTask = (tenantPath && PROBE_TASK_BY_TENANT[tenantPath]) ?? auth.probe_task; + return { + auth: { + ...(auth.api_key !== undefined && { api_key: auth.api_key }), + ...(auth.basic !== undefined && { basic: auth.basic }), + probe_task: probeTask, + }, + }; +} +/** + * Pick run-scoped transport auth for the manual storyboard runner. + * + * `security_baseline` positive credential probes use normal initialized + * transport calls, so a single run can only prove one static credential type. + * Dual-credential kits must be split into per-mechanism runs before they can + * be graded safely here. + */ +export function authForStoryboard(storyboardId, kit, defaultBearerToken) { + if (storyboardId === 'security_baseline' && kit?.auth?.api_key && kit.auth.basic) { + throw new Error('security_baseline test kit declares both auth.api_key and auth.basic; manual runner cannot grade both initialized-session credential paths in one run'); + } + if ((storyboardId === 'billing_gate_dispatch' || storyboardId === 'security_baseline') && kit?.auth?.api_key) { + return { type: 'bearer', token: kit.auth.api_key }; + } + if (storyboardId === 'security_baseline' && kit?.auth?.basic) { + const { username, password, credentials } = kit.auth.basic; + if (typeof username === 'string' && username && typeof password === 'string') { + return { type: 'basic', username, password }; + } + if (typeof credentials === 'string') { + const colonIndex = credentials.indexOf(':'); + if (colonIndex > 0) { + return { + type: 'basic', + username: credentials.slice(0, colonIndex), + password: credentials.slice(colonIndex + 1), + }; + } + } + throw new Error('security_baseline auth.basic must provide a non-empty username and a password string, or credentials in "username:password" form; the password may be empty'); + } + return { type: 'bearer', token: defaultBearerToken }; +} +//# sourceMappingURL=storyboard-runner-options.js.map \ No newline at end of file diff --git a/dist/compliance/storyboard-runner-options.js.map b/dist/compliance/storyboard-runner-options.js.map new file mode 100644 index 0000000000..63b5c1f0f9 --- /dev/null +++ b/dist/compliance/storyboard-runner-options.js.map @@ -0,0 +1 @@ +{"version":3,"file":"storyboard-runner-options.js","sourceRoot":"","sources":["../../server/src/compliance/storyboard-runner-options.ts"],"names":[],"mappings":"AAWA;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA2B;IAC1D,KAAK,EAAE,gBAAgB;IACvB,QAAQ,EAAE,gBAAgB;IAC1B,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,wBAAwB;CACrC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,GAA8B,EAC9B,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW;IAEpC,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC;IACvB,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,IAAI,EAAE,UAAU;QAAE,OAAO,SAAS,CAAC;IAC1E,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;IACrG,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,UAAU,IAAI,oBAAoB,CAAC,UAAU,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC;IACtF,OAAO;QACL,IAAI,EAAE;YACJ,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC;YAC5D,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YACtD,UAAU,EAAE,SAAS;SACtB;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,YAAoB,EACpB,GAA8B,EAC9B,kBAA0B;IAE1B,IAAI,YAAY,KAAK,mBAAmB,IAAI,GAAG,EAAE,IAAI,EAAE,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CACb,uJAAuJ,CACxJ,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,YAAY,KAAK,uBAAuB,IAAI,YAAY,KAAK,mBAAmB,CAAC,IAAI,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC7G,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;IACrD,CAAC;IAED,IAAI,YAAY,KAAK,mBAAmB,IAAI,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC7D,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QAC3D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC7E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAC/C,CAAC;QACD,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;oBAC1C,QAAQ,EAAE,WAAW,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC;iBAC5C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,6JAA6J,CAAC,CAAC;IACjL,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC;AACvD,CAAC"} \ No newline at end of file