diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0ab90bd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +# Cancel superseded runs on the same ref. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + frontend: + name: typecheck · test · build + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Typecheck + run: bun run lint + + - name: Test + run: bun test + + - name: Build + run: bun run build + env: + # A bare (root) build in CI keeps the check independent of the + # populace.dev mount path; production sets the real basePath. + NEXT_PUBLIC_BASE_PATH: "" diff --git a/.github/workflows/populace-delta-alert.yml b/.github/workflows/populace-delta-alert.yml new file mode 100644 index 0000000..c4037f4 --- /dev/null +++ b/.github/workflows/populace-delta-alert.yml @@ -0,0 +1,51 @@ +name: Populace delta alert + +# The HF webhook (app/api/hf-webhook) posts the delta the moment a release +# publishes. This scheduled run is the safety net: a weekly check that posts to +# Slack only when the latest release moved beyond a declared band vs the +# previous one (--quiet-when-clean). Also runnable on demand for an arbitrary +# pair. No-ops with a clear log line when the webhook secret is unset. + +on: + schedule: + - cron: "0 13 * * 1" # Mondays 13:00 UTC + workflow_dispatch: + inputs: + a: + description: "Earlier release id (default: previous registry release)" + required: false + b: + description: "Later release id (default: latest)" + required: false + country: + description: "Dataset (us or uk)" + required: false + default: us + +jobs: + alert: + name: compute and post release delta + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Compute and post delta + env: + SLACK_WEBHOOK_POPULACE_US: ${{ secrets.SLACK_WEBHOOK_POPULACE_US }} + SLACK_WEBHOOK_POPULACE_UK: ${{ secrets.SLACK_WEBHOOK_POPULACE_UK }} + run: | + bun run scripts/populace-delta-alert.ts \ + --country "${{ github.event.inputs.country || 'us' }}" \ + ${{ github.event.inputs.a && format('--a {0}', github.event.inputs.a) || '' }} \ + ${{ github.event.inputs.b && format('--b {0}', github.event.inputs.b) || '' }} \ + ${{ github.event_name == 'schedule' && '--quiet-when-clean' || '' }} diff --git a/docs/proactive-evidence/README.md b/docs/proactive-evidence/README.md new file mode 100644 index 0000000..29c00f1 --- /dev/null +++ b/docs/proactive-evidence/README.md @@ -0,0 +1,36 @@ +# Proactive-diagnostics evidence (#101) + +Verification artifacts for the "buildable now" slice of the proactive-diagnostics +epic ([#101](https://github.com/PolicyEngine/calibration-diagnostics/issues/101)). +All screenshots and outputs are from a local `next start` reading the **live** +`policyengine/populace-us` release resolved through `latest.json` +(`populace-us-2024-buildi-sparse-rmloss100-6e8e929-20260709T034135Z`). + +## Screenshots + +| File | View | +|---|---| +| `coverage-board.png` | New **Coverage** view — source coverage manifest (10 hard-target families, 25/41 aliases covered, 16 reviewed exclusions each linking its issue), with the input-column (populace#369) and reform-smoke (populace#368) sections in their graceful "not published for this release" state. | +| `certification-panel.png` | New **Certification** view — 13 gates (10 build-manifest + 3 side files), per-gate outcome / enforcement / evidence sha, forward-compatible with populace#381's `passed\|failed\|skipped\|waived` schema. | +| `overview-delta-banner.png` | Overview with the **"since you last looked"** banner (localStorage seeded to the prior release): 4 beyond-band flags, top-mover chips, and the flag bullets. | + +## Delta script (two real releases) + +`delta-alert-output.txt` — `bun run scripts/populace-delta-alert.ts` computing +latest vs the previous registry release, with the Slack webhook **unset** (the +no-op log path). It flags the +12.4pp within-10% jump, the −83% targets-included +drop, the changed target surface, and a real 27→25 source-coverage shrink. + +## Badge endpoints + +`badge-endpoints.txt` — the three shields.io endpoints served live: +`default-release → buildi`, `gates → 11/11 (brightgreen)`, +`within10 → 88.9% (brightgreen)`. + +## Verification (exit codes) + +``` +bun run lint (tsc --noEmit) → exit 0 +bun test → exit 0 (80 pass, 0 fail, 10 files) +bun run build (next build) → exit 0 (all new routes + pages emitted) +``` diff --git a/docs/proactive-evidence/badge-endpoints.txt b/docs/proactive-evidence/badge-endpoints.txt new file mode 100644 index 0000000..dcd721f --- /dev/null +++ b/docs/proactive-evidence/badge-endpoints.txt @@ -0,0 +1,11 @@ +# Live badge endpoints (localhost next start, real buildi release) + +GET /api/populace/badge/default-release +{"schemaVersion":1,"label":"populace-us default","message":"buildi","color":"blue","cacheSeconds":600} + +GET /api/populace/badge/gates +{"schemaVersion":1,"label":"gates","message":"11/11","color":"brightgreen","cacheSeconds":600} + +GET /api/populace/badge/within10 +{"schemaVersion":1,"label":"within 10%","message":"88.9%","color":"brightgreen","cacheSeconds":600} + diff --git a/docs/proactive-evidence/certification-panel.png b/docs/proactive-evidence/certification-panel.png new file mode 100644 index 0000000..13a901b Binary files /dev/null and b/docs/proactive-evidence/certification-panel.png differ diff --git a/docs/proactive-evidence/coverage-board.png b/docs/proactive-evidence/coverage-board.png new file mode 100644 index 0000000..67633cb Binary files /dev/null and b/docs/proactive-evidence/coverage-board.png differ diff --git a/docs/proactive-evidence/delta-alert-output.txt b/docs/proactive-evidence/delta-alert-output.txt new file mode 100644 index 0000000..5ae8c3d --- /dev/null +++ b/docs/proactive-evidence/delta-alert-output.txt @@ -0,0 +1,19 @@ + +Populace US release delta + previous: populace-us-2024-sparse-l0-refit-57k-71a0887-national-only-20260701 + latest: populace-us-2024-buildi-sparse-rmloss100-6e8e929-20260709T034135Z + +metric previous latest Δ +------------------------------------------------------------------------ +Calibration loss 0.0440 0.0308 -0.013180 (not comparable) +Within 10% of target 76.5% 88.9% +12.4pp ⚠ beyond +Records kept 57.2K 57.2K +58 (+0.1%) ok +Targets included 32.6K 5.5K -27.1K (-83.1%) ⚠ beyond + +flags: + • Within 10% of target moved 76.5% → 88.9% (Δ +12.4pp), beyond its 2pp band. + • Targets included moved 32.6K → 5.5K (Δ -27.1K (-83.1%)), beyond its 5% band. + • Target surface changed: +49 added, −27172 removed — loss not directly comparable. + • Source coverage shrank: covered 27 → 25, missing 0 → 0. + +SLACK_WEBHOOK_POPULACE_US is unset — printed only, nothing posted. diff --git a/docs/proactive-evidence/overview-delta-banner.png b/docs/proactive-evidence/overview-delta-banner.png new file mode 100644 index 0000000..c9ed2b3 Binary files /dev/null and b/docs/proactive-evidence/overview-delta-banner.png differ diff --git a/frontend/app/api/hf-webhook/route.ts b/frontend/app/api/hf-webhook/route.ts index 6328c21..707da20 100644 --- a/frontend/app/api/hf-webhook/route.ts +++ b/frontend/app/api/hf-webhook/route.ts @@ -3,7 +3,8 @@ import { timingSafeEqual } from "node:crypto"; import { NextResponse } from "next/server"; import type { PopulaceCountry } from "@/lib/populace/latest-artifact"; -import { postReleaseAlert } from "@/lib/slack"; +import { loadLatestDelta } from "@/lib/populace/deltas"; +import { postDeltaAlert, postReleaseAlert } from "@/lib/slack"; export const runtime = "nodejs"; // Push endpoint — must run on every call, never served from cache. @@ -88,5 +89,18 @@ export async function POST(request: Request) { } } - return NextResponse.json({ ok: true, country, alerted }); + // Follow the release ping with the computed headline delta (latest vs the + // previous registry release) — the epic's "on new latest.json, post the delta + // table". Best-effort and read-fresh; a failure here never fails the webhook. + let deltaPosted = false; + try { + const report = await loadLatestDelta(0, country); + if (report.available) { + deltaPosted = await postDeltaAlert({ country, report }); + } + } catch (error) { + console.error("Delta alert failed:", error); + } + + return NextResponse.json({ ok: true, country, alerted, deltaPosted }); } diff --git a/frontend/app/api/populace/badge/[metric]/route.ts b/frontend/app/api/populace/badge/[metric]/route.ts new file mode 100644 index 0000000..e829543 --- /dev/null +++ b/frontend/app/api/populace/badge/[metric]/route.ts @@ -0,0 +1,96 @@ +import { NextResponse } from "next/server"; + +import { + asObject, + hfResolveUrl, + loadPointerReleaseId, + parseCountry, + type PopulaceCountry, +} from "@/lib/populace/latest-artifact"; +import { loadSourceCoverage } from "@/lib/populace/coverage"; +import { buildCertification, type SideGateInput } from "@/lib/populace/certification"; +import { + defaultReleaseBadge, + gatesBadge, + isBadgeMetric, + within10Badge, + type Shield, +} from "@/lib/populace/badges"; + +export const revalidate = 600; +export const runtime = "nodejs"; +export const maxDuration = 60; + +// Fetch just build_manifest.json (small) — never the multi-MB diagnostics — so a +// badge stays cheap. The calibration gate carries fraction_within_10pct too. +async function fetchBuildManifest( + releaseId: string, + country: PopulaceCountry, +): Promise> { + const url = hfResolveUrl(`releases/${releaseId}/build_manifest.json`, country); + const res = await fetch(url, { next: { revalidate }, signal: AbortSignal.timeout(20_000) }); + if (!res.ok) throw new Error(`HF fetch failed ${res.status}: ${url}`); + return asObject(await res.json()); +} + +const ERROR_SHIELD: Shield = { + schemaVersion: 1, + label: "populace", + message: "unavailable", + color: "lightgrey", + cacheSeconds: 120, +}; + +export async function GET( + request: Request, + context: { params: Promise<{ metric: string }> }, +) { + const { metric } = await context.params; + const country = parseCountry(new URL(request.url).searchParams.get("country")); + if (!isBadgeMetric(metric)) { + return NextResponse.json( + { ...ERROR_SHIELD, message: "unknown badge" }, + { status: 404 }, + ); + } + try { + const { release_id: releaseId } = await loadPointerReleaseId(revalidate, country); + + if (metric === "default-release") { + return NextResponse.json(defaultReleaseBadge(releaseId, country)); + } + + if (metric === "within10") { + const manifest = await fetchBuildManifest(releaseId, country); + const calibration = asObject(asObject(manifest.gates).calibration); + const fraction = + typeof calibration.fraction_within_10pct === "number" + ? calibration.fraction_within_10pct + : null; + return NextResponse.json(within10Badge(fraction)); + } + + // metric === "gates" + const [manifest, source] = await Promise.all([ + fetchBuildManifest(releaseId, country), + loadSourceCoverage(releaseId, revalidate, country), + ]); + const sideGates: SideGateInput[] = [ + { + key: "us_source_coverage", + available: source.available, + gate: source.available ? source.gate : null, + enforced: source.available ? source.classification === "release_gate" : null, + }, + { key: "input_coverage", available: false }, + { key: "reform_coverage_smoke", available: false }, + ]; + const cert = buildCertification(manifest, releaseId, sideGates); + const ran = cert.totals.total - cert.totals.skipped; + return NextResponse.json(gatesBadge(cert.totals.passed, ran, cert.totals.failed)); + } catch { + // Return a valid shield (200) so a README badge degrades gracefully rather + // than rendering a broken image. + return NextResponse.json(ERROR_SHIELD); + } +} diff --git a/frontend/app/api/populace/certification/route.ts b/frontend/app/api/populace/certification/route.ts new file mode 100644 index 0000000..2bbaf84 --- /dev/null +++ b/frontend/app/api/populace/certification/route.ts @@ -0,0 +1,74 @@ +import { NextResponse } from "next/server"; + +import { + classifyApiError, + hfResolveUrl, + loadRelease, + parseCountry, + scrub, +} from "@/lib/populace/latest-artifact"; +import { + loadInputColumnCoverage, + loadReformSmoke, + loadSourceCoverage, +} from "@/lib/populace/coverage"; +import { buildCertification, type SideGateInput } from "@/lib/populace/certification"; + +export const revalidate = 300; +export const runtime = "nodejs"; +export const maxDuration = 120; + +export async function GET(request: Request) { + const params = new URL(request.url).searchParams; + const release = params.get("release") ?? "latest"; + const country = parseCountry(params.get("country")); + try { + const cal = await loadRelease(release, revalidate, country); + const releaseId = cal.release_id; + const [source, inputColumns, reformSmoke] = await Promise.all([ + loadSourceCoverage(releaseId, revalidate, country), + loadInputColumnCoverage(releaseId, revalidate, country), + loadReformSmoke(releaseId, revalidate, country), + ]); + + const sideGates: SideGateInput[] = [ + { + key: "us_source_coverage", + available: source.available, + gate: source.available ? source.gate : null, + // classification "release_gate" means this gate blocks publication. + enforced: source.available ? source.classification === "release_gate" : null, + reviewed_exclusions: source.available ? source.reviewed_exclusions : [], + }, + { + key: "input_coverage", + available: inputColumns.available, + gate: inputColumns.available ? inputColumns.gate : null, + enforced: inputColumns.available ? inputColumns.enforced : null, + }, + { + key: "reform_coverage_smoke", + available: reformSmoke.available, + gate: reformSmoke.available ? reformSmoke.gate : null, + enforced: reformSmoke.available ? reformSmoke.enforced : null, + }, + ]; + + const certification = buildCertification(cal.build_manifest, releaseId, sideGates); + const prefix = `releases/${releaseId}`; + return NextResponse.json( + scrub({ + release_id: releaseId, + updated_at: cal.updated_at, + certification, + source_artifacts: [ + { name: "build_manifest", path: `${prefix}/build_manifest.json`, url: hfResolveUrl(`${prefix}/build_manifest.json`, country) }, + { name: "us_source_coverage", path: `${prefix}/us_source_coverage.json`, url: hfResolveUrl(`${prefix}/us_source_coverage.json`, country) }, + ], + }), + ); + } catch (error) { + const { status, body } = classifyApiError(error); + return NextResponse.json(body, { status }); + } +} diff --git a/frontend/app/api/populace/coverage/route.ts b/frontend/app/api/populace/coverage/route.ts new file mode 100644 index 0000000..a92b3f9 --- /dev/null +++ b/frontend/app/api/populace/coverage/route.ts @@ -0,0 +1,47 @@ +import { NextResponse } from "next/server"; + +import { + classifyApiError, + loadPointerReleaseId, + parseCountry, + scrub, +} from "@/lib/populace/latest-artifact"; +import { + loadInputColumnCoverage, + loadReformSmoke, + loadSourceCoverage, +} from "@/lib/populace/coverage"; + +export const revalidate = 300; +export const runtime = "nodejs"; +export const maxDuration = 120; + +export async function GET(request: Request) { + const params = new URL(request.url).searchParams; + const release = params.get("release") ?? "latest"; + const country = parseCountry(params.get("country")); + try { + // Resolve the pointer once so the three artifacts read the same release and + // don't each re-fetch latest.json. + const releaseId = + release && release !== "latest" + ? release + : (await loadPointerReleaseId(revalidate, country)).release_id; + const [source, inputColumns, reformSmoke] = await Promise.all([ + loadSourceCoverage(releaseId, revalidate, country), + loadInputColumnCoverage(releaseId, revalidate, country), + loadReformSmoke(releaseId, revalidate, country), + ]); + return NextResponse.json( + scrub({ + release_id: releaseId, + source, + input_columns: inputColumns, + reform_smoke: reformSmoke, + }), + ); + } catch (error) { + const { status, body } = classifyApiError(error); + return NextResponse.json(body, { status }); + } +} diff --git a/frontend/app/api/populace/deltas/latest/route.ts b/frontend/app/api/populace/deltas/latest/route.ts new file mode 100644 index 0000000..4992771 --- /dev/null +++ b/frontend/app/api/populace/deltas/latest/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; + +import { classifyApiError, parseCountry, scrub } from "@/lib/populace/latest-artifact"; +import { loadLatestDelta, loadReleaseDelta } from "@/lib/populace/deltas"; + +export const revalidate = 300; +export const runtime = "nodejs"; +export const maxDuration = 120; + +// The release-delta feed: the same payload the Slack alert posts and the +// "since you last looked" banner reads. Defaults to latest vs previous; ?a=&b= +// diffs an arbitrary pair. +export async function GET(request: Request) { + const params = new URL(request.url).searchParams; + const a = params.get("a"); + const b = params.get("b"); + const country = parseCountry(params.get("country")); + try { + const report = + a && b + ? await loadReleaseDelta(a, b, revalidate, country) + : await loadLatestDelta(revalidate, country); + return NextResponse.json(scrub(report)); + } catch (error) { + const { status, body } = classifyApiError(error); + return NextResponse.json(body, { status }); + } +} diff --git a/frontend/app/populace/certification/page.tsx b/frontend/app/populace/certification/page.tsx new file mode 100644 index 0000000..4151900 --- /dev/null +++ b/frontend/app/populace/certification/page.tsx @@ -0,0 +1,10 @@ +import { AppShell } from "@/components/layout/app-shell"; +import { PopulaceCertificationView } from "@/components/populace/populace-certification-view"; + +export default function PopulaceCertificationPage() { + return ( + + + + ); +} diff --git a/frontend/app/populace/coverage/page.tsx b/frontend/app/populace/coverage/page.tsx new file mode 100644 index 0000000..e4e6360 --- /dev/null +++ b/frontend/app/populace/coverage/page.tsx @@ -0,0 +1,10 @@ +import { AppShell } from "@/components/layout/app-shell"; +import { PopulaceCoverageView } from "@/components/populace/populace-coverage-view"; + +export default function PopulaceCoveragePage() { + return ( + + + + ); +} diff --git a/frontend/components/layout/nav-sidebar.tsx b/frontend/components/layout/nav-sidebar.tsx index aab0f09..7fe337d 100644 --- a/frontend/components/layout/nav-sidebar.tsx +++ b/frontend/components/layout/nav-sidebar.tsx @@ -27,6 +27,7 @@ const NAV_GROUPS: { label: string; items: NavItem[] }[] = [ label: "Dataset accuracy", items: [ { href: "/populace", label: "Calibration fit", also: ["/populace/targets"] }, + { href: "/populace/coverage", label: "Coverage", usOnly: true }, { href: "/populace/reforms", label: "External checks", usOnly: true }, { href: "/populace/model-coverage", label: "Validation reach", usOnly: true }, { href: "/populace/datasets", label: "Cross-dataset", usOnly: true }, @@ -35,6 +36,7 @@ const NAV_GROUPS: { label: string; items: NavItem[] }[] = [ { label: "Releases", items: [ + { href: "/populace/certification", label: "Certification", usOnly: true }, { href: "/populace/compare", label: "Compare versions" }, { href: "/populace/staging", label: "Staging candidates", usOnly: true }, ], diff --git a/frontend/components/populace/delta-banner.tsx b/frontend/components/populace/delta-banner.tsx new file mode 100644 index 0000000..c68f40f --- /dev/null +++ b/frontend/components/populace/delta-banner.tsx @@ -0,0 +1,139 @@ +"use client"; + +import { useEffect, useState } from "react"; + +import { withBasePath } from "@/lib/base-path"; +import { fmtCompact, releaseLabel } from "@/components/shared/format"; +import { StatusPill } from "@/components/shared/status-pill"; +import { usePopulaceDeltasLatest, type MetricDelta } from "@/lib/api/hooks/use-populace"; + +const LAST_SEEN_KEY = "populace:last-seen-release"; + +function moverText(m: MetricDelta): string { + if (m.abs_delta == null) return "—"; + const sign = m.abs_delta > 0 ? "+" : ""; + if (m.unit === "share") return `${sign}${(m.abs_delta * 100).toFixed(1)}pp`; + if (m.unit === "count") { + const rel = m.rel_delta != null ? ` (${sign}${(m.rel_delta * 100).toFixed(1)}%)` : ""; + return `${sign}${fmtCompact(m.abs_delta)}${rel}`; + } + return `${sign}${Math.abs(m.abs_delta) < 1 ? m.abs_delta.toFixed(4) : m.abs_delta.toExponential(2)}`; +} + +// "Since you last looked" — remembers the release id the reader last acknowledged +// (localStorage) and, on a newer latest release, surfaces the top movers and any +// beyond-band flags. Silent on first visit and once acknowledged. +export function DeltaBanner() { + const { data } = usePopulaceDeltasLatest(); + // undefined = not yet read from storage; null = never set. + const [lastSeen, setLastSeen] = useState(undefined); + + useEffect(() => { + try { + setLastSeen(window.localStorage.getItem(LAST_SEEN_KEY)); + } catch { + setLastSeen(null); + } + }, []); + + const latestRelease = data?.available ? data.b_release : null; + + // First visit: record the current release silently so the banner only fires + // on genuinely newer releases from here on. + useEffect(() => { + if (latestRelease && lastSeen === null) { + try { + window.localStorage.setItem(LAST_SEEN_KEY, latestRelease); + } catch { + // ignore storage failures — the banner just won't persist. + } + setLastSeen(latestRelease); + } + }, [latestRelease, lastSeen]); + + if (!data?.available || lastSeen === undefined || lastSeen === null) return null; + if (lastSeen === data.b_release) return null; + + function markSeen() { + if (!latestRelease) return; + try { + window.localStorage.setItem(LAST_SEEN_KEY, latestRelease); + } catch { + // ignore + } + setLastSeen(latestRelease); + } + + const loud = data.flags.length > 0; + const movers = data.top_movers.slice(0, 3); + + return ( +
+
+
+
+ Since you last looked + {loud ? ( + + {data.flags.length} beyond-band flag{data.flags.length === 1 ? "" : "s"} + + ) : ( + all within band + )} +
+
+ New release {releaseLabel(data.b_release, data.b_date)}{" "} + (you last saw {releaseLabel(lastSeen)}). +
+
+
+ + View diff → + + +
+
+ + {movers.length > 0 && ( +
+ {movers.map((m) => ( + + {m.label} + + {moverText(m)} + + + ))} +
+ )} + + {loud && ( +
    + {data.flags.slice(0, 4).map((flag) => ( +
  • {flag}
    • + ))} +
+ )} +
+ ); +} diff --git a/frontend/components/populace/populace-certification-view.tsx b/frontend/components/populace/populace-certification-view.tsx new file mode 100644 index 0000000..830f6e8 --- /dev/null +++ b/frontend/components/populace/populace-certification-view.tsx @@ -0,0 +1,254 @@ +"use client"; + +import { useMemo, useState } from "react"; + +import { EmptyState } from "@/components/shared/empty-state"; +import { fmt, releaseLabel } from "@/components/shared/format"; +import { KpiCard } from "@/components/shared/kpi-card"; +import { LoadingBlock } from "@/components/shared/LoadingBlock"; +import { PageHeader } from "@/components/shared/page-header"; +import { SectionCard } from "@/components/shared/section-card"; +import { StatusPill, type StatusTone } from "@/components/shared/status-pill"; +import { ToolbarSelect } from "@/components/shared/toolbar-select"; +import { + releaseSelectOptions, + usePopulaceCertification, + usePopulaceReleases, + type CertificationGate, + type CoverageExclusion, + type CoverageIssueRef, + type GateOutcome, +} from "@/lib/api/hooks/use-populace"; + +const OUTCOME_TONE: Record = { + passed: "success", + failed: "danger", + waived: "warning", + skipped: "neutral", + unknown: "neutral", +}; + +const SOURCE_LABEL: Record = { + build_manifest: "build manifest", + us_source_coverage: "side file", + input_coverage: "side file", + reform_coverage_smoke: "side file", +}; + +function IssueLinks({ issues }: { issues: CoverageIssueRef[] }) { + if (!issues.length) { + return no issue; + } + return ( + + {issues.map((ref) => ( + + {ref.repo === "populace" ? `#${ref.number}` : `${ref.repo}#${ref.number}`} + + ))} + + ); +} + +function ExclusionRegister({ + gateLabel, + entries, +}: { + gateLabel: string; + entries: CoverageExclusion[]; +}) { + return ( +
+
+ {gateLabel} · {entries.length} +
+
    + {entries.map((entry) => ( +
  • +
    + {entry.subject} + +
    +

    {entry.reason}

    +
    • + ))} +
+
+ ); +} + +function GateRow({ gate }: { gate: CertificationGate }) { + return ( + + +
{gate.label}
+ {gate.summary && ( +
{gate.summary}
+ )} + {gate.failures.length > 0 && ( +
    + {gate.failures.slice(0, 4).map((failure, i) => ( +
  • {failure}
    • + ))} +
+ )} + + + {gate.outcome} + + + {gate.enforced == null ? ( + not declared + ) : ( + + {gate.enforced ? "enforced" : "advisory"} + + )} + + + {gate.evidence_sha ? gate.evidence_sha.slice(0, 12) : "—"} + + {SOURCE_LABEL[gate.source]} + + ); +} + +export function PopulaceCertificationView() { + const { data: releaseData } = usePopulaceReleases(); + const [release, setRelease] = useState(""); + const { data, isLoading, error } = usePopulaceCertification(release || undefined); + const releaseOptions = useMemo(() => releaseSelectOptions(releaseData), [releaseData]); + + const cert = data?.certification; + const totals = cert?.totals; + + return ( +
+ + } + /> + + {isLoading ? ( + + ) : error || !cert || !totals ? ( + + ) : ( + <> +
+ + + + +
+ + + Evidence read live from{" "} + {data.source_artifacts.map((artifact, i) => ( + + {i > 0 ? " · " : ""} + + {artifact.name} + + + ))} + . + + ) : null + } + > +
+ + + + + + + + + + + + {cert.gates.map((gate) => ( + + ))} + +
GateOutcomeEnforcementEvidence shaFrom
+
+ + + + {cert.reviewed_exclusion_registers.length ? ( +
+ {cert.reviewed_exclusion_registers.map((register) => ( + + ))} +
+ ) : ( + + )} + + +
+ Certifying release {releaseLabel(cert.release_id)} + {data?.updated_at ? ` · published ${data.updated_at}` : ""}. +
+ + )} +
+ ); +} diff --git a/frontend/components/populace/populace-compare-view.tsx b/frontend/components/populace/populace-compare-view.tsx index f16a4cb..7dffb99 100644 --- a/frontend/components/populace/populace-compare-view.tsx +++ b/frontend/components/populace/populace-compare-view.tsx @@ -562,11 +562,14 @@ export function PopulaceCompareView() { const [a, setA] = useState(""); const [b, setB] = useState(""); - // Default B to the latest release and A to the next one down. + // Deep links (the "since you last looked" banner and the Slack delta alert) + // pass ?a=&b= to preselect the two releases; otherwise default B to the latest + // release and A to the next one down. useEffect(() => { if (!releases.length) return; - if (!b) setB(releaseData?.latest_release_id || releases[0].release_id); - if (!a && releases[1]) setA(releases[1].release_id); + const params = new URLSearchParams(window.location.search); + if (!b) setB(params.get("b") || releaseData?.latest_release_id || releases[0].release_id); + if (!a) setA(params.get("a") || releases[1]?.release_id || ""); }, [releases, releaseData, a, b]); const { data, isLoading, error } = usePopulaceCompare(a, b); diff --git a/frontend/components/populace/populace-coverage-view.tsx b/frontend/components/populace/populace-coverage-view.tsx new file mode 100644 index 0000000..8a2b074 --- /dev/null +++ b/frontend/components/populace/populace-coverage-view.tsx @@ -0,0 +1,510 @@ +"use client"; + +import { Fragment, useMemo, useState } from "react"; + +import { EmptyState } from "@/components/shared/empty-state"; +import { fmt, fmtCompact, releaseLabel } from "@/components/shared/format"; +import { KpiCard } from "@/components/shared/kpi-card"; +import { LoadingBlock } from "@/components/shared/LoadingBlock"; +import { PageHeader } from "@/components/shared/page-header"; +import { SectionCard } from "@/components/shared/section-card"; +import { StatusPill } from "@/components/shared/status-pill"; +import { ToolbarSelect } from "@/components/shared/toolbar-select"; +import { + releaseSelectOptions, + usePopulaceCoverage, + usePopulaceReleases, + type CoverageExclusion, + type CoverageIssueRef, + type HardTargetFamily, +} from "@/lib/api/hooks/use-populace"; + +const POPULACE_369 = "https://github.com/PolicyEngine/populace/issues/369"; +const POPULACE_368 = "https://github.com/PolicyEngine/populace/issues/368"; + +// A reviewed exclusion must name a live issue (#286 "cannot rot"). We link +// every issue the reason references so a reviewer can check it in one click. +function IssueLinks({ issues }: { issues: CoverageIssueRef[] }) { + if (!issues.length) { + return ( + + no issue + + ); + } + return ( + + {issues.map((ref) => ( + e.stopPropagation()} + className="inline-flex items-center rounded-full border border-border bg-muted/40 px-2 py-0.5 font-mono text-[11px] text-primary hover:underline" + > + {ref.repo === "populace" ? `#${ref.number}` : `${ref.repo}#${ref.number}`} + + ))} + + ); +} + +function ExclusionList({ exclusions }: { exclusions: CoverageExclusion[] }) { + if (!exclusions.length) return null; + return ( +
    + {exclusions.map((exclusion) => ( +
  • +
    + {exclusion.subject} + +
    +

    {exclusion.reason}

    +
    • + ))} +
+ ); +} + +const STATE_TONE: Record = { + covered: "success", + partial: "warning", + excluded: "warning", + missing: "danger", +}; + +const STATE_LABEL: Record = { + covered: "covered", + partial: "partial", + excluded: "reviewed-excluded", + missing: "missing", +}; + +function FamilyTable({ families }: { families: HardTargetFamily[] }) { + const [open, setOpen] = useState>(new Set()); + function toggle(key: string) { + setOpen((current) => { + const next = new Set(current); + if (next.has(key)) next.delete(key); + else next.add(key); + return next; + }); + } + return ( +
+ + + + + + + + + + + + + {families.map((family) => { + const isOpen = open.has(family.key); + return ( + + toggle(family.key)} + className="cursor-pointer border-b border-border/60 last:border-b-0 hover:bg-muted/30" + > + + + + + + + + {isOpen && ( + + + + )} + + ); + })} + +
Source familyRequiredCoveredExcludedMissingState
+
+ + ▸ + + {family.label} +
+ {family.key} + 		{family.required.length}{family.covered.length}{family.reviewed_exclusions.length} + {family.missing.length} + + {STATE_LABEL[family.state]} +
+
+ {family.covered.length > 0 && ( +
+
+ Covered ({family.covered.length}) +
+
+ {family.covered.map((alias) => ( + + {alias} + + ))} +
+
+ )} + {family.missing.length > 0 && ( +
+
+ Missing — required but absent ({family.missing.length}) +
+
+ {family.missing.map((alias) => ( + + {alias} + + ))} +
+
+ )} + {family.reviewed_exclusions.length > 0 && ( +
+
+ Reviewed exclusions ({family.reviewed_exclusions.length}) +
+ +
+ )} +
+
+
+ ); +} + +export function PopulaceCoverageView() { + const { data: releaseData } = usePopulaceReleases(); + const [release, setRelease] = useState(""); + const { data, isLoading, error } = usePopulaceCoverage(release || undefined); + const releaseOptions = useMemo(() => releaseSelectOptions(releaseData), [releaseData]); + + const source = data?.source; + const inputColumns = data?.input_columns; + const reformSmoke = data?.reform_smoke; + + return ( +
+ + } + /> + + {isLoading ? ( + + ) : error ? ( + + ) : ( + <> + {source?.available && source.summary ? ( + <> +
+ + + + +
+ + + Hard-target source coverage + {source.gate?.passed != null && ( + + gate {source.gate.passed ? "passed" : "failed"} + + )} + + } + description="Each source family the release is required to cover. Click a family to see the exact package aliases it covers, any that are missing, and reviewed fiscal-refresh exclusions with their tracking issues." + padded={false} + footer={ + source.artifact ? ( + <> + Read live from{" "} + + {source.artifact.path} + + {source.ledger_commit ? ` · ledger ${source.ledger_commit.slice(0, 10)}` : ""} + + ) : null + } + > + + + + {(source.validation_only_families?.length ?? 0) > 0 && ( + + + + + + + + + + + {source.validation_only_families?.map((family) => ( + + + + + + ))} + +
FamilyPackagesActivated
+ {family.label} + {family.key} + + {family.required.join(", ") || "—"} + + + {family.activated ? "activated" : "validation only"} + +
+ + )} + + {(source.source_gap_families?.length ?? 0) > 0 && ( + + + + + + + + + + {source.source_gap_families?.map((family) => ( + + + + + ))} + +
FamilyMissing source packages
+ {family.label} + +
    + {family.missing_source_packages.map((pkg) => ( +
  • {pkg}
    • + ))} +
+
+ + )} + + ) : ( + + )} + + {/* populace#369 — the per-column eCPS input coverage gate. */} + + Input-column coverage + {inputColumns?.available && inputColumns.enforced != null && ( + + {inputColumns.enforced ? "enforced gate" : "advisory"} + + )} + + } + description="Every input column the reference eCPS exports, required present and non-degenerate (populace#369)." + padded={inputColumns?.available ? false : true} + > + {inputColumns?.available && inputColumns.summary ? ( +
+ + + + + + + + + + + {[...(inputColumns.required ?? []), ...(inputColumns.reviewed_exclusions ?? [])].map( + (column) => ( + + + + + + + ), + )} + +
ColumnPresentSignalIssue
{column.column} + + {column.present === false ? "absent" : "present"} + + + + {column.degenerate ? "degenerate" : "varies"} + + + +
+
+ ) : ( + + Track populace#369 → + + } + /> + )} + + + {/* populace#368 — reform-coverage smoke: does a bound reform score? */} + + Reform-coverage smoke + {reformSmoke?.available && reformSmoke.enforced != null && ( + + {reformSmoke.enforced ? "enforced gate" : "advisory"} + + )} + + } + description="Pinned reform probes that must score nonzero where the policy mechanically binds. A $0 on a bound reform is the SSI-scores-$0 failure class (populace#368) — surfaced here before an analyst burns a day on it." + padded={reformSmoke?.available ? false : true} + > + {reformSmoke?.available && reformSmoke.summary ? ( +
+ + + + + + + + + + + {reformSmoke.probes?.map((probe) => ( + + + + + + + ))} + +
ProbeScored valueVerdictIssue
+ {probe.name} + {probe.description && ( +

{probe.description}

+ )} + 		+ {probe.scored_value == null ? "—" : fmtCompact(probe.scored_value)} + + + {probe.verdict === "scored" ? "scores" : probe.verdict === "zero" ? "scores $0" : "unknown"} + + + +
+
+ ) : ( + + Track populace#368 → + + } + /> + )} + + +
+ Current release:{" "} + + {data ? releaseLabel(data.release_id) : "—"} + +
+ + )} +
+ ); +} diff --git a/frontend/components/populace/populace-overview-view.tsx b/frontend/components/populace/populace-overview-view.tsx index 15d74c6..53ce390 100644 --- a/frontend/components/populace/populace-overview-view.tsx +++ b/frontend/components/populace/populace-overview-view.tsx @@ -3,6 +3,7 @@ import { useMemo, useState } from "react"; import { CalibrationMap } from "@/components/populace/calibration-map"; +import { DeltaBanner } from "@/components/populace/delta-banner"; import { useCountry } from "@/components/layout/country-context"; import { withBasePath } from "@/lib/base-path"; import { EmptyState } from "@/components/shared/empty-state"; @@ -137,6 +138,9 @@ export function PopulaceOverviewView() { } /> + {/* Only meaningful for the live latest release, not a historical pick. */} + {!release && } +
+ apiGet("/populace/coverage", { release: release || undefined, country }), + staleTime: 5 * 60 * 1000, + }); +} + +// --- certification panel ---------------------------------------------------- + +export type GateOutcome = "passed" | "failed" | "skipped" | "waived" | "unknown"; + +export interface CertificationGate { + key: string; + label: string; + outcome: GateOutcome; + enforced: boolean | null; + evidence_sha: string | null; + failure_count: number; + failures: string[]; + reviewed_exclusions: CoverageExclusion[]; + stale_exclusions: string[]; + summary: string | null; + source: "build_manifest" | "us_source_coverage" | "input_coverage" | "reform_coverage_smoke"; +} + +export interface CertificationResponse { + release_id: string; + updated_at: string | null; + certification: { + release_id: string; + gates: CertificationGate[]; + totals: { + total: number; + passed: number; + failed: number; + skipped: number; + waived: number; + enforced: number; + }; + reviewed_exclusion_registers: { + gate_key: string; + gate_label: string; + entries: CoverageExclusion[]; + }[]; + stale_exclusion_count: number; + }; + source_artifacts: { name: string; path: string; url: string }[]; +} + +export function usePopulaceCertification(release?: string) { + const { country } = useCountry(); + return useQuery({ + queryKey: ["populace", "certification", country, release ?? "latest"], + queryFn: () => + apiGet("/populace/certification", { + release: release || undefined, + country, + }), + staleTime: 5 * 60 * 1000, + }); +} + +// --- release deltas --------------------------------------------------------- + +export type BandVerdict = "within" | "beyond" | null; + +export interface MetricDelta { + key: string; + label: string; + family: string | null; + unit: "loss" | "share" | "count"; + a: number | null; + b: number | null; + abs_delta: number | null; + rel_delta: number | null; + improve: "better" | "worse" | "flat" | null; + band: BandVerdict; + threshold: number | null; + comparable: boolean; + note: string | null; +} + +export interface ReformDelta { + id: string; + name: string; + category: string | null; + in_sample: boolean; + a_abs_rel: number | null; + b_abs_rel: number | null; + delta: number | null; + band: BandVerdict; +} + +export interface DeltaReport { + available: true; + a_release: string; + b_release: string; + a_date: string; + b_date: string; + generated_at: string; + headline: MetricDelta[]; + reforms: ReformDelta[]; + reforms_available: boolean; + surfaces_differ: boolean; + surface: { added: number; removed: number; improved: number; regressed: number }; + coverage_delta: { + a_covered: number; + b_covered: number; + a_missing: number; + b_missing: number; + a_reviewed_excluded: number; + b_reviewed_excluded: number; + shrank: boolean; + } | null; + gate_changes: { key: string; label: string; a_outcome: string; b_outcome: string; kind: string }[]; + flags: string[]; + top_movers: MetricDelta[]; +} + +export interface DeltaUnavailable { + available: false; + reason: string; + a_release: string | null; + b_release: string | null; +} + +export type DeltasResponse = DeltaReport | DeltaUnavailable; + +export function usePopulaceDeltasLatest() { + const { country } = useCountry(); + return useQuery({ + queryKey: ["populace", "deltas", "latest", country], + queryFn: () => apiGet("/populace/deltas/latest", { country }), + staleTime: 5 * 60 * 1000, + }); +} diff --git a/frontend/lib/populace/badges.test.ts b/frontend/lib/populace/badges.test.ts new file mode 100644 index 0000000..8f9bc4d --- /dev/null +++ b/frontend/lib/populace/badges.test.ts @@ -0,0 +1,48 @@ +import { expect, test } from "bun:test"; + +import { + defaultReleaseBadge, + gatesBadge, + isBadgeMetric, + releaseBuildSlug, + within10Badge, +} from "./badges"; + +test("releaseBuildSlug pulls the build name out of a release id", () => { + expect( + releaseBuildSlug("populace-us-2024-buildi-sparse-rmloss100-6e8e929-20260709T034135Z"), + ).toBe("buildi"); + expect(releaseBuildSlug("populace-us-2024-f0af251-703bd81a565c-20260620T201958Z")).toBe( + "f0af251", + ); +}); + +test("default-release badge is a valid shield", () => { + const badge = defaultReleaseBadge("populace-us-2024-buildi-x-20260709T034135Z", "us"); + expect(badge.schemaVersion).toBe(1); + expect(badge.label).toBe("populace-us default"); + expect(badge.message).toBe("buildi"); + expect(badge.color).toBe("blue"); +}); + +test("gates badge colors green only when all ran gates passed", () => { + expect(gatesBadge(11, 11, 0).color).toBe("brightgreen"); + expect(gatesBadge(10, 11, 0).color).toBe("yellow"); + expect(gatesBadge(9, 11, 2).color).toBe("red"); + expect(gatesBadge(11, 11, 0).message).toBe("11/11"); +}); + +test("within10 badge formats the share and grades the color", () => { + expect(within10Badge(0.8888).message).toBe("88.9%"); + expect(within10Badge(0.8888).color).toBe("brightgreen"); + expect(within10Badge(0.75).color).toBe("yellow"); + expect(within10Badge(0.5).color).toBe("orange"); + expect(within10Badge(null).message).toBe("unknown"); +}); + +test("isBadgeMetric guards the route parameter", () => { + expect(isBadgeMetric("gates")).toBe(true); + expect(isBadgeMetric("within10")).toBe(true); + expect(isBadgeMetric("default-release")).toBe(true); + expect(isBadgeMetric("nonsense")).toBe(false); +}); diff --git a/frontend/lib/populace/badges.ts b/frontend/lib/populace/badges.ts new file mode 100644 index 0000000..42d6ca2 --- /dev/null +++ b/frontend/lib/populace/badges.ts @@ -0,0 +1,64 @@ +// Shields.io-compatible badge payloads for READMEs and papers, e.g. +// ![](https://img.shields.io/endpoint?url=/api/populace/badge/gates) +// The endpoint schema is { schemaVersion: 1, label, message, color }. + +export interface Shield { + schemaVersion: 1; + label: string; + message: string; + color: string; + cacheSeconds?: number; +} + +export type BadgeMetric = "default-release" | "gates" | "within10"; + +export const BADGE_METRICS: BadgeMetric[] = ["default-release", "gates", "within10"]; + +export function isBadgeMetric(value: string): value is BadgeMetric { + return (BADGE_METRICS as string[]).includes(value); +} + +// The build slug that names a release, e.g. +// "populace-us-2024-buildi-sparse-…-20260709T034135Z" → "buildi". +export function releaseBuildSlug(releaseId: string): string { + const stripped = releaseId.replace(/^populace-[a-z]{2}-\d{4}-/, ""); + const token = stripped.split("-")[0]; + return token || releaseId; +} + +export function defaultReleaseBadge(releaseId: string, country: string): Shield { + return { + schemaVersion: 1, + label: `populace-${country.toLowerCase()} default`, + message: releaseBuildSlug(releaseId), + color: "blue", + cacheSeconds: 600, + }; +} + +// `ran` is the number of gates that actually executed (total minus skipped), so +// an unpublished side-file gate never drags the badge down. +export function gatesBadge(passed: number, ran: number, failed: number): Shield { + return { + schemaVersion: 1, + label: "gates", + message: `${passed}/${ran}`, + color: failed > 0 ? "red" : passed < ran ? "yellow" : "brightgreen", + cacheSeconds: 600, + }; +} + +export function within10Badge(fraction: number | null): Shield { + if (fraction == null || !Number.isFinite(fraction)) { + return { schemaVersion: 1, label: "within 10%", message: "unknown", color: "lightgrey", cacheSeconds: 600 }; + } + const pct = fraction * 100; + const color = pct >= 85 ? "brightgreen" : pct >= 70 ? "yellow" : "orange"; + return { + schemaVersion: 1, + label: "within 10%", + message: `${pct.toFixed(1)}%`, + color, + cacheSeconds: 600, + }; +} diff --git a/frontend/lib/populace/certification.test.ts b/frontend/lib/populace/certification.test.ts new file mode 100644 index 0000000..8f7f3c0 --- /dev/null +++ b/frontend/lib/populace/certification.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from "bun:test"; + +import { buildCertification, normalizeGate } from "./certification"; + +// Trimmed real build_manifest.gates shape (the buildi release): most gates are +// { passed, failures, details }; target_compilation carries no `passed`. +const GATES = { + calibration: { + passed: true, + failures: [], + final_loss: 0.03083, + fraction_within_10pct: 0.8888, + }, + target_compilation: { + declared_targets: 5514, + compiled_candidate_targets: 5514, + dropped_target_names: [], + target_frame_checkpoint: { identity_sha256: "b2eb410149e9ad16060fdbb9bb" }, + }, + degenerate_input_signal: { + passed: true, + failures: [], + details: { + columns_checked: 73, + reviewed_exclusions: { + s_corp_income: "Combined partnership/S-corp income is carried in partnership_income; see #359.", + second_home_mortgage_balance: "Second-home decomposition not imputed; tracked in populace#340.", + }, + stale_exclusions: [], + }, + }, +}; + +test("a normal gate maps passed:true to outcome 'passed'", () => { + const gate = normalizeGate("calibration", GATES.calibration, "build_manifest"); + expect(gate.outcome).toBe("passed"); + expect(gate.summary).toContain("within 10%"); + // enforcement is not declared in today's build manifest. + expect(gate.enforced).toBeNull(); +}); + +test("target_compilation (no `passed`) is derived from dropped/compiled counts", () => { + const gate = normalizeGate( + "target_compilation", + { ...GATES.target_compilation, details: GATES.target_compilation }, + "build_manifest", + ); + expect(gate.outcome).toBe("passed"); + // the checkpoint identity sha is surfaced as evidence. + expect(gate.evidence_sha).toBe("b2eb410149e9ad16060fdbb9bb"); +}); + +test("target_compilation with dropped targets fails", () => { + const raw = { + declared_targets: 100, + compiled_candidate_targets: 100, + dropped_target_names: ["x/y/z"], + details: { + declared_targets: 100, + compiled_candidate_targets: 100, + dropped_target_names: ["x/y/z"], + }, + }; + expect(normalizeGate("target_compilation", raw, "build_manifest").outcome).toBe("failed"); +}); + +test("reviewed exclusions are parsed with their linked issues", () => { + const gate = normalizeGate("degenerate_input_signal", GATES.degenerate_input_signal, "build_manifest"); + expect(gate.reviewed_exclusions).toHaveLength(2); + const sCorp = gate.reviewed_exclusions.find((e) => e.subject === "s_corp_income")!; + expect(sCorp.issues[0].number).toBe(359); +}); + +test("populace#381 fields (explicit outcome/enforced/evidence_sha) win when present", () => { + const gate = normalizeGate( + "base_population_scale", + { + outcome: "waived", + enforced: false, + evidence_sha: "deadbeefcafe0001", + passed: true, // #381 outcome must override the legacy boolean + failures: [], + }, + "build_manifest", + ); + expect(gate.outcome).toBe("waived"); + expect(gate.enforced).toBe(false); + expect(gate.evidence_sha).toBe("deadbeefcafe0001"); +}); + +test("a skipped/waived status maps to the right outcome", () => { + expect(normalizeGate("g", { status: "skipped_checkpoint_hit" }, "build_manifest").outcome).toBe("skipped"); + expect(normalizeGate("g", { status: "waived_by_flag" }, "build_manifest").outcome).toBe("waived"); +}); + +test("stale reviewed exclusions (#286 cannot-rot) are surfaced", () => { + const gate = normalizeGate( + "health_input_signal", + { passed: true, details: { unused_exclusions: ["takes_up_aca_if_eligible"] } }, + "build_manifest", + ); + expect(gate.stale_exclusions).toEqual(["takes_up_aca_if_eligible"]); +}); + +test("buildCertification folds side files and tallies totals", () => { + const cert = buildCertification({ gates: GATES }, "rel-1", [ + { + key: "us_source_coverage", + available: true, + gate: { passed: true, failures: [] }, + enforced: true, + reviewed_exclusions: [ + { subject: "census-acs-national", reason: "Reviewed exclusion, Issue #40.", issues: [] }, + ], + }, + { key: "input_coverage", available: false }, + { key: "reform_coverage_smoke", available: false }, + ]); + + // 3 manifest gates + 3 side gates. + expect(cert.totals.total).toBe(6); + expect(cert.totals.passed).toBe(4); // calibration, target_compilation, degenerate, source_coverage + // the two unpublished side files are honestly skipped. + expect(cert.totals.skipped).toBe(2); + expect(cert.totals.enforced).toBe(1); + // registers group reviewed exclusions by gate. + const gateKeys = cert.reviewed_exclusion_registers.map((r) => r.gate_key).sort(); + expect(gateKeys).toEqual(["degenerate_input_signal", "us_source_coverage"]); +}); diff --git a/frontend/lib/populace/certification.ts b/frontend/lib/populace/certification.ts new file mode 100644 index 0000000..fcfc7d9 --- /dev/null +++ b/frontend/lib/populace/certification.ts @@ -0,0 +1,325 @@ +// Certification panel data layer. Gate *execution* (what blocked the build) and +// gate *certification* (what the published evidence proves) are different +// contracts (populace#381). This module normalizes every gate the release +// carries — the build_manifest.gates map plus the coverage/smoke side files — +// into one verdict surface: name, outcome, whether it was enforced, and the +// evidence sha where present. +// +// Today's manifests express a gate as `{ passed: bool, failures, details }`. +// populace#381 will publish an exhaustive report with an explicit +// `outcome ∈ {passed,failed,skipped,waived}`, an `enforced` flag, and an +// `evidence_sha`. normalizeGate reads the richer fields when they exist and +// derives them from the current shape when they don't, so #381 drops in without +// restructuring the panel. + +import { asObject } from "./latest-artifact"; +import { exclusionsFromMap, type CoverageExclusion } from "./coverage"; + +type JsonObject = Record; + +// Compact loss for a one-line gate summary: small normalized losses read best +// with a few decimals, larger raw objectives as exponential. +function fmtLossValue(value: number): string { + if (value === 0) return "0"; + if (Math.abs(value) < 1) return value.toFixed(value < 0.001 ? 6 : 4); + return value.toExponential(2).replace("e+", "e"); +} + +export type GateOutcome = "passed" | "failed" | "skipped" | "waived" | "unknown"; + +export type GateSource = + | "build_manifest" + | "us_source_coverage" + | "input_coverage" + | "reform_coverage_smoke"; + +export interface CertificationGate { + key: string; + label: string; + outcome: GateOutcome; + // null = the artifact does not declare enforcement (today's build_manifest + // gates); populace#381 makes this an explicit bool. + enforced: boolean | null; + evidence_sha: string | null; + failure_count: number; + failures: string[]; + reviewed_exclusions: CoverageExclusion[]; + // #286 "cannot rot": an exclusion whose column has caught up is stale and + // should fail the gate. Surfaced so a reviewer sees rot immediately. + stale_exclusions: string[]; + summary: string | null; + source: GateSource; +} + +export interface ReviewedExclusionRegister { + gate_key: string; + gate_label: string; + entries: CoverageExclusion[]; +} + +export interface Certification { + release_id: string; + gates: CertificationGate[]; + totals: { + total: number; + passed: number; + failed: number; + skipped: number; + waived: number; + enforced: number; + }; + reviewed_exclusion_registers: ReviewedExclusionRegister[]; + stale_exclusion_count: number; +} + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value ? value : null; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.map((v) => (typeof v === "string" ? v : JSON.stringify(v))) + : []; +} + +const GATE_LABELS: Record = { + calibration: "Calibration", + target_compilation: "Target compilation", + target_profile_coverage: "Target profile coverage", + base_population_scale: "Base population scale", + health_input_signal: "Health input signal", + immigration_composition: "Immigration composition", + degenerate_input_signal: "Degenerate input signal", + ecps_parity: "eCPS parity", + hours_worked_signal: "Hours-worked signal", + snap_take_up_signal: "SNAP take-up signal", + us_source_coverage: "Source coverage", + input_coverage: "Input-column coverage", + reform_coverage_smoke: "Reform-coverage smoke", +}; + +function gateLabel(key: string): string { + if (GATE_LABELS[key]) return GATE_LABELS[key]; + return key + .split("_") + .map((word) => (word.length <= 3 ? word.toUpperCase() : word[0].toUpperCase() + word.slice(1))) + .join(" "); +} + +const KNOWN_OUTCOMES = new Set(["passed", "failed", "skipped", "waived"]); + +// The evidence sha lives under a different key depending on the gate; check the +// ones producers actually emit, plus the target-frame checkpoint's identity. +function evidenceSha(raw: JsonObject, details: JsonObject): string | null { + const checkpoint = asObject(details.target_frame_checkpoint); + return ( + stringOrNull(raw.evidence_sha) ?? + stringOrNull(raw.sha256) ?? + stringOrNull(raw.identity_sha256) ?? + stringOrNull(details.sha256) ?? + stringOrNull(checkpoint.identity_sha256) ?? + null + ); +} + +function deriveOutcome(key: string, raw: JsonObject, details: JsonObject): GateOutcome { + // populace#381: explicit outcome wins. + const explicit = stringOrNull(raw.outcome); + if (explicit && KNOWN_OUTCOMES.has(explicit as GateOutcome)) { + return explicit as GateOutcome; + } + const status = (stringOrNull(raw.status) ?? "").toLowerCase(); + if (status.includes("waiv")) return "waived"; + if (status.includes("skip")) return "skipped"; + if (typeof raw.passed === "boolean") return raw.passed ? "passed" : "failed"; + // target_compilation carries no `passed`; it compiled iff nothing was dropped + // and every declared target became a candidate. + if (key === "target_compilation") { + const declared = numberOrNull(details.declared_targets); + const compiled = numberOrNull(details.compiled_candidate_targets); + const dropped = Array.isArray(details.dropped_target_names) + ? details.dropped_target_names.length + : 0; + if (declared != null && compiled != null) { + return dropped === 0 && compiled >= declared ? "passed" : "failed"; + } + } + const failures = stringArray(raw.failures ?? details.failures); + if (failures.length > 0) return "failed"; + return "unknown"; +} + +// A short human line so a gate scans without expanding its details. +function gateSummary(key: string, details: JsonObject): string | null { + switch (key) { + case "calibration": { + const loss = numberOrNull(details.final_loss); + const within = numberOrNull(details.fraction_within_10pct); + const parts = [ + loss != null ? `loss ${fmtLossValue(loss)}` : null, + within != null ? `${(within * 100).toFixed(1)}% within 10%` : null, + ].filter(Boolean); + return parts.length ? parts.join(" · ") : null; + } + case "target_compilation": { + const declared = numberOrNull(details.declared_targets); + const compiled = numberOrNull(details.compiled_candidate_targets); + const dropped = Array.isArray(details.dropped_target_names) + ? details.dropped_target_names.length + : 0; + if (declared == null && compiled == null) return null; + return `${compiled ?? "?"}/${declared ?? "?"} compiled · ${dropped} dropped`; + } + case "target_profile_coverage": { + const reqs = numberOrNull(details.requirements_checked); + const targets = numberOrNull(details.targets_checked); + if (reqs == null && targets == null) return null; + return `${reqs ?? "?"} requirements · ${targets ?? "?"} targets`; + } + case "base_population_scale": { + const pop = numberOrNull(details.population); + const bench = numberOrNull(details.benchmark); + const rel = numberOrNull(details.relative_error); + if (pop == null || bench == null) return null; + return `${(pop / 1e6).toFixed(1)}M vs ${(bench / 1e6).toFixed(1)}M${rel != null ? ` (${(rel * 100).toFixed(1)}%)` : ""}`; + } + case "ecps_parity": { + const gaps = numberOrNull(details.gaps); + const exempted = Array.isArray(details.exempted) ? details.exempted.length : null; + if (gaps == null) return null; + return `${gaps} gaps${exempted != null ? ` · ${exempted} exempted` : ""}`; + } + case "snap_take_up_signal": { + const share = numberOrNull(details.take_up_share); + return share != null ? `take-up ${(share * 100).toFixed(1)}%` : null; + } + case "hours_worked_signal": { + const share = numberOrNull(details.worked_share); + return share != null ? `worked share ${(share * 100).toFixed(1)}%` : null; + } + default: + return null; + } +} + +// #286 stale exclusions across the gate's several possible key names. +function staleExclusions(details: JsonObject): string[] { + return [ + ...stringArray(details.stale_exclusions), + ...stringArray(details.unused_reviewed_exclusions), + ...stringArray(details.unused_exclusions), + ]; +} + +export function normalizeGate( + key: string, + raw: JsonObject, + source: GateSource, +): CertificationGate { + // Gates are inconsistent about nesting: calibration/target_compilation put + // their metrics at the top level, base_population_scale et al. under a + // `details` object. Merge both (top level wins) so every reader resolves the + // field wherever the producer happened to put it. + const fields = { ...asObject(raw.details), ...raw }; + const failures = stringArray(fields.failures); + const reviewed = exclusionsFromMap(fields.reviewed_exclusions); + return { + key, + label: gateLabel(key), + outcome: deriveOutcome(key, raw, fields), + enforced: typeof raw.enforced === "boolean" ? raw.enforced : null, + evidence_sha: evidenceSha(raw, fields), + failure_count: failures.length, + failures, + reviewed_exclusions: reviewed, + stale_exclusions: staleExclusions(fields), + summary: gateSummary(key, fields), + source, + }; +} + +// Side files (us_source_coverage / input_coverage / reform_coverage_smoke) are +// not represented in build_manifest.gates yet (populace#381). Fold them in so +// the certification surface is exhaustive: a published side file becomes a gate; +// a missing one becomes an honest "skipped — not published" entry. +export interface SideGateInput { + key: GateSource; + available: boolean; + gate?: { passed: boolean | null; failures: string[] } | null; + enforced?: boolean | null; + reviewed_exclusions?: CoverageExclusion[]; +} + +function normalizeSideGate(input: SideGateInput): CertificationGate { + if (!input.available) { + return { + key: input.key, + label: gateLabel(input.key), + outcome: "skipped", + enforced: input.enforced ?? null, + evidence_sha: null, + failure_count: 0, + failures: [], + reviewed_exclusions: [], + stale_exclusions: [], + summary: "not published for this release", + source: input.key, + }; + } + const passed = input.gate?.passed ?? null; + const failures = input.gate?.failures ?? []; + return { + key: input.key, + label: gateLabel(input.key), + outcome: passed == null ? "unknown" : passed ? "passed" : "failed", + enforced: input.enforced ?? null, + evidence_sha: null, + failure_count: failures.length, + failures, + reviewed_exclusions: input.reviewed_exclusions ?? [], + stale_exclusions: [], + summary: null, + source: input.key, + }; +} + +export function buildCertification( + buildManifest: JsonObject, + releaseId: string, + sideGates: SideGateInput[] = [], +): Certification { + const gatesRaw = asObject(buildManifest.gates); + const manifestGates = Object.entries(gatesRaw).map(([key, value]) => + normalizeGate(key, asObject(value), "build_manifest"), + ); + const gates = [...manifestGates, ...sideGates.map(normalizeSideGate)]; + + const totals = { + total: gates.length, + passed: gates.filter((g) => g.outcome === "passed").length, + failed: gates.filter((g) => g.outcome === "failed").length, + skipped: gates.filter((g) => g.outcome === "skipped").length, + waived: gates.filter((g) => g.outcome === "waived").length, + enforced: gates.filter((g) => g.enforced === true).length, + }; + + const reviewed_exclusion_registers: ReviewedExclusionRegister[] = gates + .filter((g) => g.reviewed_exclusions.length > 0) + .map((g) => ({ + gate_key: g.key, + gate_label: g.label, + entries: g.reviewed_exclusions, + })); + + return { + release_id: releaseId, + gates, + totals, + reviewed_exclusion_registers, + stale_exclusion_count: gates.reduce((sum, g) => sum + g.stale_exclusions.length, 0), + }; +} diff --git a/frontend/lib/populace/coverage.test.ts b/frontend/lib/populace/coverage.test.ts new file mode 100644 index 0000000..dcc885f --- /dev/null +++ b/frontend/lib/populace/coverage.test.ts @@ -0,0 +1,173 @@ +import { expect, test } from "bun:test"; + +import { + buildInputColumnCoverage, + buildReformSmoke, + buildSourceCoverage, +} from "./coverage"; + +// A trimmed slice of a real releases//us_source_coverage.json (the buildi +// release) so the parser is pinned to the shape actually on Hugging Face. +const SOURCE_FIXTURE = { + schema_version: 1, + classification: "release_gate", + source_contract: { name: "us_source_coverage", ledger_commit: "e2fc882c35f9203c" }, + gate: { name: "us_source_coverage", passed: true, failures: [] }, + coverage_summary: { + hard_target: { + families: 2, + package_aliases: 6, + covered_package_aliases: 3, + missing_package_aliases: 0, + reviewed_excluded_package_aliases: 3, + }, + validation_only: { families: 1, activated_families: 0 }, + source_gap: { families: 1, missing_source_packages: 2 }, + }, + hard_target_families: { + population_age_sex: { + label: "Population by age and sex", + package_aliases: ["census-pep-national", "census-acs-national", "census-acs-cd"], + covered_package_aliases: ["census-pep-national"], + missing_package_aliases: [], + reviewed_exclusions: { + "census-acs-national": + "Reviewed fiscal-refresh exclusion: recalibrates the Issue #40 fiscal surface only.", + "census-acs-cd": + "Reviewed fiscal-refresh exclusion: recalibrates the Issue #40 fiscal surface only.", + }, + }, + irs_soi: { + label: "SOI filer income", + package_aliases: ["soi-1-1", "soi-1-2", "soi-2-1"], + covered_package_aliases: ["soi-1-1", "soi-1-2"], + missing_package_aliases: [], + reviewed_exclusions: { + "soi-2-1": "Reviewed fiscal-refresh exclusion tracked in populace#359.", + }, + }, + }, + validation_only_families: { + snap_local_proxy: { + label: "SNAP CD proxy", + package_aliases: ["census-acs-snap-cd"], + activated_as_hard_target: false, + }, + }, + source_gap_families: { + hud_assisted_housing: { + label: "Housing assistance controls", + missing_source_packages: ["HUD Picture of Subsidized Households", "HUD unit-count tables"], + }, + }, + reviewed_exclusions: { + "census-acs-national": "Reviewed fiscal-refresh exclusion: Issue #40.", + }, + fiscal_target_support_exclusions: [ + { + source_record_id: "census_stc.fy2024.individual_income_tax.tn", + reason: "Tennessee has no modeled 2024 state income tax; cannot estimate. See #359.", + }, + ], +}; + +test("buildSourceCoverage parses families, exclusions, and their issues", () => { + const cov = buildSourceCoverage(SOURCE_FIXTURE, "rel-1", "us"); + expect(cov.available).toBe(true); + expect(cov.summary.hard_target_families).toBe(2); + expect(cov.summary.covered_aliases).toBe(3); + expect(cov.summary.reviewed_excluded_aliases).toBe(3); + + const pop = cov.hard_target_families.find((f) => f.key === "population_age_sex")!; + expect(pop.required).toHaveLength(3); + expect(pop.covered).toEqual(["census-pep-national"]); + // Covered one alias + reviewed-excluded the rest, none missing → "partial". + expect(pop.state).toBe("partial"); + expect(pop.reviewed_exclusions).toHaveLength(2); + // Every reviewed exclusion carries its tracking issue (#40). + expect(pop.reviewed_exclusions[0].issues[0].number).toBe(40); +}); + +test("a family with a missing required alias is state=missing", () => { + const withMissing = { + ...SOURCE_FIXTURE, + hard_target_families: { + broken: { + label: "Broken", + package_aliases: ["a", "b"], + covered_package_aliases: ["a"], + missing_package_aliases: ["b"], + reviewed_exclusions: {}, + }, + }, + }; + const cov = buildSourceCoverage(withMissing, "rel-1", "us"); + expect(cov.hard_target_families[0].state).toBe("missing"); +}); + +test("fiscal-support exclusions parse their record id, reason, and issue", () => { + const cov = buildSourceCoverage(SOURCE_FIXTURE, "rel-1", "us"); + expect(cov.fiscal_support_exclusions).toHaveLength(1); + const first = cov.fiscal_support_exclusions[0]; + expect(first.subject).toContain("census_stc"); + expect(first.issues[0].number).toBe(359); +}); + +test("the source-coverage artifact path is bound to the release id", () => { + const cov = buildSourceCoverage(SOURCE_FIXTURE, "rel-xyz", "us"); + expect(cov.artifact.path).toBe("releases/rel-xyz/us_source_coverage.json"); +}); + +test("buildReformSmoke derives 'scores $0' from a zero scored value", () => { + const smoke = buildReformSmoke( + { + enforced: true, + gate: { passed: false, failures: ["ssi_asset_limits scored $0"] }, + probes: [ + { name: "SSI asset limits $10k/$20k", reform: "ssi_asset_limits", scored_value: 0, issue: "populace#368" }, + { name: "EITC repeal", reform: "eitc_repeal", scored_value: 71_000_000_000 }, + ], + }, + "rel-1", + "us", + ); + expect(smoke.summary.probes).toBe(2); + expect(smoke.summary.zero).toBe(1); + expect(smoke.summary.scored).toBe(1); + const ssi = smoke.probes.find((p) => p.reform === "ssi_asset_limits")!; + expect(ssi.verdict).toBe("zero"); + expect(ssi.issues[0].number).toBe(368); + expect(smoke.enforced).toBe(true); +}); + +test("buildReformSmoke honours an explicit passed flag and a 'reforms' array", () => { + const smoke = buildReformSmoke( + { reforms: [{ name: "probe", passed: true }] }, + "rel-1", + "us", + ); + expect(smoke.probes[0].verdict).toBe("scored"); +}); + +test("buildInputColumnCoverage flags absent and degenerate required columns", () => { + const cov = buildInputColumnCoverage( + { + enforced: true, + required: [ + { column: "bank_account_assets", present: false }, + { column: "employment_income", present: true, degenerate: false }, + { column: "stock_assets", present: true, degenerate: true }, + ], + reviewed_exclusions: [ + { column: "s_corp_income", reason: "Carried in partnership_income, see #359." }, + ], + }, + "rel-1", + "us", + ); + expect(cov.summary.required).toBe(3); + expect(cov.summary.reviewed_exclusion).toBe(1); + // absent + degenerate both count as failing. + expect(cov.summary.failing).toBe(2); + expect(cov.reviewed_exclusions[0].issues[0].number).toBe(359); +}); diff --git a/frontend/lib/populace/coverage.ts b/frontend/lib/populace/coverage.ts new file mode 100644 index 0000000..08417d7 --- /dev/null +++ b/frontend/lib/populace/coverage.ts @@ -0,0 +1,489 @@ +// Reform-coverage board data layer. Three artifacts answer "what inputs does +// this release actually carry, and which reforms score vs silently default": +// +// us_source_coverage.json — PUBLISHED on main today. Per source family, the +// package aliases the release is required to cover, +// which it covers, and which are carried as reviewed +// fiscal-refresh exclusions (each naming its issue). +// input_coverage.json — populace#369, the per-column eCPS coverage gate. +// Not published on today's releases (RED-by-design +// until the SCF asset stage lands) — rendered as a +// graceful "not published" state until it ships. +// reform_coverage_smoke.json— populace#368, the pinned reform probes that must +// score nonzero where the policy mechanically binds +// (first probe: SSI asset limits $10k/$20k, the +// scores-$0 failure class). Also not yet published. +// +// Everything reads live from Hugging Face. Missing is an expected state, not a +// transport error, so each loader distinguishes the two. + +import { + asObject, + assertSafeReleaseId, + hfResolveUrl, + loadPointerReleaseId, + type PopulaceCountry, +} from "./latest-artifact"; +import { extractIssueRefs, type IssueRef } from "./issue-links"; + +type JsonObject = Record; + +export const US_SOURCE_COVERAGE_FILE = "us_source_coverage.json"; +export const INPUT_COVERAGE_FILE = "input_coverage.json"; +export const REFORM_SMOKE_FILE = "reform_coverage_smoke.json"; + +function stringOrNull(value: unknown): string | null { + return typeof value === "string" && value ? value : null; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((v): v is string => typeof v === "string") + : []; +} + +// 404 is "not published for this release" (an expected state); any other +// non-OK is an upstream failure the caller should see. +async function fetchReleaseArtifact( + releaseId: string, + filename: string, + revalidate: number, + country: PopulaceCountry, +): Promise { + const url = hfResolveUrl(`releases/${assertSafeReleaseId(releaseId)}/${filename}`, country); + const res = await fetch(url, { + next: { revalidate }, + signal: AbortSignal.timeout(20_000), + }); + if (res.status === 404) return null; + if (!res.ok) throw new Error(`HF fetch failed ${res.status}: ${url}`); + return asObject(await res.json()); +} + +async function resolveReleaseId( + releaseId: string, + revalidate: number, + country: PopulaceCountry, +): Promise { + if (releaseId && releaseId !== "latest") return releaseId; + return (await loadPointerReleaseId(revalidate, country)).release_id; +} + +// --- reviewed exclusions ---------------------------------------------------- + +export interface CoverageExclusion { + subject: string; // the package alias / column / record the exclusion covers + reason: string; + issues: IssueRef[]; // tracking issues parsed from the reason text +} + +// A {subject: reason} map into structured exclusions with their linked issues. +export function exclusionsFromMap(map: unknown): CoverageExclusion[] { + return Object.entries(asObject(map)) + .map(([subject, reason]) => ({ + subject, + reason: typeof reason === "string" ? reason : String(reason), + issues: extractIssueRefs(typeof reason === "string" ? reason : ""), + })) + .sort((a, b) => a.subject.localeCompare(b.subject)); +} + +// --- us_source_coverage.json (published) ------------------------------------ + +export type FamilyCoverageState = "covered" | "partial" | "excluded" | "missing"; + +export interface HardTargetFamily { + key: string; + label: string; + required: string[]; + covered: string[]; + missing: string[]; + reviewed_exclusions: CoverageExclusion[]; + state: FamilyCoverageState; +} + +export interface ValidationOnlyFamily { + key: string; + label: string; + required: string[]; + activated: boolean; +} + +export interface SourceGapFamily { + key: string; + label: string; + missing_source_packages: string[]; +} + +export interface SourceCoverage { + available: true; + release_id: string; + schema_version: unknown; + classification: string | null; + gate: { name: string | null; passed: boolean | null; failures: string[] }; + ledger_commit: string | null; + summary: { + hard_target_families: number; + required_aliases: number; + covered_aliases: number; + missing_aliases: number; + reviewed_excluded_aliases: number; + validation_only_families: number; + validation_only_activated: number; + source_gap_families: number; + missing_source_packages: number; + }; + hard_target_families: HardTargetFamily[]; + validation_only_families: ValidationOnlyFamily[]; + source_gap_families: SourceGapFamily[]; + reviewed_exclusions: CoverageExclusion[]; + fiscal_support_exclusions: CoverageExclusion[]; + artifact: { path: string; url: string }; +} + +export interface CoverageMissing { + available: false; + release_id: string; + reason: string; + expected_path: string; +} + +function familyState(family: HardTargetFamily): FamilyCoverageState { + if (family.missing.length > 0) return "missing"; + if (family.reviewed_exclusions.length > 0 && family.covered.length === 0) { + return "excluded"; + } + if (family.reviewed_exclusions.length > 0) return "partial"; + return "covered"; +} + +export function buildSourceCoverage( + raw: JsonObject, + releaseId: string, + country: PopulaceCountry, +): SourceCoverage { + const summary = asObject(raw.coverage_summary); + const hardSummary = asObject(summary.hard_target); + const validationSummary = asObject(summary.validation_only); + const gapSummary = asObject(summary.source_gap); + const gate = asObject(raw.gate); + + const hardTargetFamilies: HardTargetFamily[] = Object.entries( + asObject(raw.hard_target_families), + ) + .map(([key, value]) => { + const family = asObject(value); + const partial: HardTargetFamily = { + key, + label: stringOrNull(family.label) ?? key, + required: stringArray(family.package_aliases), + covered: stringArray(family.covered_package_aliases), + missing: stringArray(family.missing_package_aliases), + reviewed_exclusions: exclusionsFromMap(family.reviewed_exclusions), + state: "covered", + }; + return { ...partial, state: familyState(partial) }; + }) + .sort((a, b) => a.label.localeCompare(b.label)); + + const validationOnlyFamilies: ValidationOnlyFamily[] = Object.entries( + asObject(raw.validation_only_families), + ) + .map(([key, value]) => { + const family = asObject(value); + return { + key, + label: stringOrNull(family.label) ?? key, + required: stringArray(family.package_aliases), + activated: family.activated_as_hard_target === true, + }; + }) + .sort((a, b) => a.label.localeCompare(b.label)); + + const sourceGapFamilies: SourceGapFamily[] = Object.entries( + asObject(raw.source_gap_families), + ) + .map(([key, value]) => { + const family = asObject(value); + return { + key, + label: stringOrNull(family.label) ?? key, + missing_source_packages: stringArray(family.missing_source_packages), + }; + }) + .sort((a, b) => a.label.localeCompare(b.label)); + + const fiscalSupport = Array.isArray(raw.fiscal_target_support_exclusions) + ? (raw.fiscal_target_support_exclusions as JsonObject[]).map((entry) => { + const row = asObject(entry); + const reason = stringOrNull(row.reason) ?? ""; + return { + subject: stringOrNull(row.source_record_id) ?? "—", + reason, + issues: extractIssueRefs(reason), + }; + }) + : []; + + const path = `releases/${releaseId}/${US_SOURCE_COVERAGE_FILE}`; + return { + available: true, + release_id: releaseId, + schema_version: raw.schema_version ?? null, + classification: stringOrNull(raw.classification), + gate: { + name: stringOrNull(gate.name), + passed: typeof gate.passed === "boolean" ? gate.passed : null, + failures: stringArray(gate.failures), + }, + ledger_commit: stringOrNull(asObject(raw.source_contract).ledger_commit), + summary: { + hard_target_families: numberOrNull(hardSummary.families) ?? hardTargetFamilies.length, + required_aliases: numberOrNull(hardSummary.package_aliases) ?? 0, + covered_aliases: numberOrNull(hardSummary.covered_package_aliases) ?? 0, + missing_aliases: numberOrNull(hardSummary.missing_package_aliases) ?? 0, + reviewed_excluded_aliases: + numberOrNull(hardSummary.reviewed_excluded_package_aliases) ?? 0, + validation_only_families: + numberOrNull(validationSummary.families) ?? validationOnlyFamilies.length, + validation_only_activated: numberOrNull(validationSummary.activated_families) ?? 0, + source_gap_families: numberOrNull(gapSummary.families) ?? sourceGapFamilies.length, + missing_source_packages: numberOrNull(gapSummary.missing_source_packages) ?? 0, + }, + hard_target_families: hardTargetFamilies, + validation_only_families: validationOnlyFamilies, + source_gap_families: sourceGapFamilies, + reviewed_exclusions: exclusionsFromMap(raw.reviewed_exclusions), + fiscal_support_exclusions: fiscalSupport, + artifact: { path, url: hfResolveUrl(path, country) }, + }; +} + +export async function loadSourceCoverage( + releaseId: string, + revalidate: number, + country: PopulaceCountry = "us", +): Promise { + const id = await resolveReleaseId(releaseId, revalidate, country); + const raw = await fetchReleaseArtifact(id, US_SOURCE_COVERAGE_FILE, revalidate, country); + if (!raw) { + return { + available: false, + release_id: id, + reason: `No ${US_SOURCE_COVERAGE_FILE} published for this release yet.`, + expected_path: `releases/${id}/${US_SOURCE_COVERAGE_FILE}`, + }; + } + return buildSourceCoverage(raw, id, country); +} + +// --- input_coverage.json (populace#369, forward-compatible) ----------------- + +export interface InputColumn { + column: string; + present: boolean | null; + degenerate: boolean | null; + reason: string | null; + issues: IssueRef[]; +} + +export interface InputColumnCoverage { + available: true; + release_id: string; + enforced: boolean | null; + gate: { passed: boolean | null; failures: string[] }; + summary: { required: number; reviewed_exclusion: number; failing: number }; + required: InputColumn[]; + reviewed_exclusions: InputColumn[]; + artifact: { path: string; url: string }; +} + +// Permissive normalizer: #369 fixes the exact field names, but the board only +// needs column / present / degenerate / reason, under whichever nesting ships. +function normalizeColumn(value: unknown): InputColumn { + const row = asObject(value); + const reason = stringOrNull(row.reason) ?? stringOrNull(row.note); + return { + column: stringOrNull(row.column) ?? stringOrNull(row.name) ?? "—", + present: typeof row.present === "boolean" ? row.present : null, + degenerate: + typeof row.degenerate === "boolean" + ? row.degenerate + : typeof row.constant_at_default === "boolean" + ? row.constant_at_default + : null, + reason, + issues: extractIssueRefs(reason ?? ""), + }; +} + +export function buildInputColumnCoverage( + raw: JsonObject, + releaseId: string, + country: PopulaceCountry, +): InputColumnCoverage { + const columns = asObject(raw.columns); + const requiredRaw = Array.isArray(raw.required) + ? raw.required + : Array.isArray(columns.required) + ? columns.required + : []; + const exclusionRaw = Array.isArray(raw.reviewed_exclusions) + ? raw.reviewed_exclusions + : Array.isArray(columns.reviewed_exclusion) + ? columns.reviewed_exclusion + : []; + const required = requiredRaw.map(normalizeColumn); + const reviewed = exclusionRaw.map(normalizeColumn); + const gate = asObject(raw.gate); + const failing = required.filter((c) => c.present === false || c.degenerate === true).length; + const path = `releases/${releaseId}/${INPUT_COVERAGE_FILE}`; + return { + available: true, + release_id: releaseId, + enforced: typeof raw.enforced === "boolean" ? raw.enforced : null, + gate: { + passed: typeof gate.passed === "boolean" ? gate.passed : null, + failures: stringArray(gate.failures), + }, + summary: { + required: required.length, + reviewed_exclusion: reviewed.length, + failing, + }, + required, + reviewed_exclusions: reviewed, + artifact: { path, url: hfResolveUrl(path, country) }, + }; +} + +export async function loadInputColumnCoverage( + releaseId: string, + revalidate: number, + country: PopulaceCountry = "us", +): Promise { + const id = await resolveReleaseId(releaseId, revalidate, country); + const raw = await fetchReleaseArtifact(id, INPUT_COVERAGE_FILE, revalidate, country); + if (!raw) { + return { + available: false, + release_id: id, + reason: `No ${INPUT_COVERAGE_FILE} published for this release yet (populace#369 — the per-column eCPS coverage gate).`, + expected_path: `releases/${id}/${INPUT_COVERAGE_FILE}`, + }; + } + return buildInputColumnCoverage(raw, id, country); +} + +// --- reform_coverage_smoke.json (populace#368, forward-compatible) ---------- + +export interface ReformProbe { + name: string; + reform: string | null; + scored_value: number | null; + verdict: "scored" | "zero" | "unknown"; + passed: boolean | null; + description: string | null; + issues: IssueRef[]; +} + +export interface ReformSmoke { + available: true; + release_id: string; + enforced: boolean | null; + gate: { passed: boolean | null; failures: string[] }; + summary: { probes: number; scored: number; zero: number }; + probes: ReformProbe[]; + artifact: { path: string; url: string }; +} + +function scoredValueOf(row: JsonObject): number | null { + return ( + numberOrNull(row.scored_value) ?? + numberOrNull(row.budget_effect) ?? + numberOrNull(row.value) ?? + numberOrNull(row.score) ?? + null + ); +} + +function probeVerdict(row: JsonObject, scored: number | null): ReformProbe["verdict"] { + const explicit = stringOrNull(row.verdict) ?? stringOrNull(row.band) ?? stringOrNull(row.status); + if (explicit) { + const v = explicit.toLowerCase(); + if (v.includes("zero") || v.includes("default") || v.includes("fail")) return "zero"; + if (v.includes("scored") || v.includes("nonzero") || v.includes("pass")) return "scored"; + } + if (typeof row.passed === "boolean") return row.passed ? "scored" : "zero"; + if (scored != null) return Math.abs(scored) > 0 ? "scored" : "zero"; + return "unknown"; +} + +function normalizeProbe(value: unknown): ReformProbe { + const row = asObject(value); + const scored = scoredValueOf(row); + const description = stringOrNull(row.description) ?? stringOrNull(row.reason); + const issueText = [description, stringOrNull(row.issue)].filter(Boolean).join(" "); + return { + name: stringOrNull(row.name) ?? stringOrNull(row.id) ?? stringOrNull(row.reform) ?? "—", + reform: stringOrNull(row.reform) ?? stringOrNull(row.id), + scored_value: scored, + verdict: probeVerdict(row, scored), + passed: typeof row.passed === "boolean" ? row.passed : null, + description, + issues: extractIssueRefs(issueText), + }; +} + +export function buildReformSmoke( + raw: JsonObject, + releaseId: string, + country: PopulaceCountry, +): ReformSmoke { + const list = Array.isArray(raw.probes) + ? raw.probes + : Array.isArray(raw.reforms) + ? raw.reforms + : []; + const probes = list.map(normalizeProbe); + const gate = asObject(raw.gate); + const path = `releases/${releaseId}/${REFORM_SMOKE_FILE}`; + return { + available: true, + release_id: releaseId, + enforced: typeof raw.enforced === "boolean" ? raw.enforced : null, + gate: { + passed: typeof gate.passed === "boolean" ? gate.passed : null, + failures: stringArray(gate.failures), + }, + summary: { + probes: probes.length, + scored: probes.filter((p) => p.verdict === "scored").length, + zero: probes.filter((p) => p.verdict === "zero").length, + }, + probes, + artifact: { path, url: hfResolveUrl(path, country) }, + }; +} + +export async function loadReformSmoke( + releaseId: string, + revalidate: number, + country: PopulaceCountry = "us", +): Promise { + const id = await resolveReleaseId(releaseId, revalidate, country); + const raw = await fetchReleaseArtifact(id, REFORM_SMOKE_FILE, revalidate, country); + if (!raw) { + return { + available: false, + release_id: id, + reason: `No ${REFORM_SMOKE_FILE} published for this release yet (populace#368 — the reform-coverage smoke that fails a bound reform scoring $0).`, + expected_path: `releases/${id}/${REFORM_SMOKE_FILE}`, + }; + } + return buildReformSmoke(raw, id, country); +} diff --git a/frontend/lib/populace/delta-bands.json b/frontend/lib/populace/delta-bands.json new file mode 100644 index 0000000..09a7c6b --- /dev/null +++ b/frontend/lib/populace/delta-bands.json @@ -0,0 +1,8 @@ +{ + "_comment": "Per-metric-family bands for the release delta alert. A move is flagged beyond-band when |absolute delta| exceeds `abs` (for share/loss metrics, in the metric's own units — e.g. 0.02 = 2 percentage points) OR |relative delta| exceeds `rel` (for count/level metrics). Tune per family; the alert and the in-page banner both read these.", + "final_loss": { "family": "calibration", "abs": 0.02, "rel": 0.25 }, + "fraction_within_10pct": { "family": "calibration", "abs": 0.02 }, + "n_nonzero": { "family": "population", "rel": 0.05 }, + "included_target_count": { "family": "population", "rel": 0.05 }, + "reform_validation": { "family": "reform", "abs": 0.05 } +} diff --git a/frontend/lib/populace/deltas.test.ts b/frontend/lib/populace/deltas.test.ts new file mode 100644 index 0000000..6690597 --- /dev/null +++ b/frontend/lib/populace/deltas.test.ts @@ -0,0 +1,246 @@ +import { expect, test } from "bun:test"; + +import { + computeReleaseDelta, + deltaSlackPayload, + formatDeltaTable, + type ComputeDeltaInput, +} from "./deltas"; +import type { Calibration } from "./latest-artifact"; +import type { ReformValidation } from "./reforms"; +import type { Certification } from "./certification"; +import type { SourceCoverage } from "./coverage"; + +function cal(id: string, over: Partial): Calibration { + return { + source: "huggingface_live", + release_id: id, + updated_at: null, + schema_version: 2, + weight_entity: "household", + options: {}, + l0_lambda: null, + n_nonzero: 100000, + n_records: 100000, + initial_loss: 1, + final_loss: 0.03, + loss_kind: "normalized_target_loss", + fraction_within_10pct: 0.88, + loss_trajectory: [], + skipped: [], + declared_targets: 100, + compiled_candidate_targets: 100, + dropped_target_names: [], + included_target_count: 100, + build_manifest: {}, + release_manifest: {}, + rows: [], + ...over, + }; +} + +function reforms(rows: { id: string; name: string; absRel: number | null }[]): ReformValidation { + return { + available: true, + source: "huggingface_live", + release_id: "r", + updated_at: null, + schema_version: 1, + baseline_period: 2026, + scoring_window: "FY2027", + rows: rows.map((r) => ({ + id: r.id, + name: r.name, + category: "OBBBA", + description: null, + in_sample: false, + period: 2026, + jct_score: 1, + jct_score_fy2026: 1, + jct_score_type: "conventional", + jct_window: "FY2027", + jct_benchmark_window: "FY2027", + jct_source: null, + jct_source_url: null, + jct_published: null, + populace_estimate: 1, + populace_window: "FY2027", + populace_annual: null, + abs_error: null, + relative_error: r.absRel, + abs_relative_error: r.absRel, + within_10pct: r.absRel == null ? null : r.absRel <= 0.1, + direction: null, + })), + summary: { + n_reforms: rows.length, + n_scored: rows.length, + within_10pct: 0, + mean_abs_relative_error: null, + median_abs_relative_error: null, + n_out_of_sample: rows.length, + n_out_of_sample_scored: rows.length, + out_of_sample_within_10pct: 0, + out_of_sample_mean_abs_relative_error: null, + }, + }; +} + +const NOW = new Date("2026-07-09T00:00:00Z"); + +function run(input: Omit) { + return computeReleaseDelta({ ...input, now: NOW }); +} + +test("a loss move beyond its band is flagged and marked 'worse'", () => { + const report = run({ + a: cal("populace-us-2024-aaa-20260701", { final_loss: 0.03 }), + b: cal("populace-us-2024-bbb-20260709", { final_loss: 0.08 }), + }); + const loss = report.headline.find((m) => m.key === "final_loss")!; + expect(loss.abs_delta).toBeCloseTo(0.05, 6); + expect(loss.band).toBe("beyond"); + expect(loss.improve).toBe("worse"); + expect(report.flags.some((f) => f.startsWith("Calibration loss moved"))).toBe(true); +}); + +test("a small loss move stays within band", () => { + const report = run({ + a: cal("a", { final_loss: 0.03 }), + b: cal("b", { final_loss: 0.031 }), + }); + expect(report.headline.find((m) => m.key === "final_loss")!.band).toBe("within"); +}); + +test("within-10% improvement reads as 'better'", () => { + const report = run({ + a: cal("a", { fraction_within_10pct: 0.88 }), + b: cal("b", { fraction_within_10pct: 0.889 }), + }); + const w = report.headline.find((m) => m.key === "fraction_within_10pct")!; + expect(w.improve).toBe("better"); + expect(w.band).toBe("within"); +}); + +test("a records drop beyond the 5% band is flagged via relative delta", () => { + const report = run({ + a: cal("a", { n_nonzero: 100000 }), + b: cal("b", { n_nonzero: 90000 }), + }); + const records = report.headline.find((m) => m.key === "n_nonzero")!; + expect(records.rel_delta).toBeCloseTo(-0.1, 6); + expect(records.band).toBe("beyond"); +}); + +test("a changed target surface makes loss non-comparable and is flagged", () => { + const report = run({ + a: cal("a", { rows: [{ name: "t1", base_name: "t1", target: 1, final_estimate: 1 }] }), + b: cal("b", { rows: [] }), + }); + expect(report.surfaces_differ).toBe(true); + const loss = report.headline.find((m) => m.key === "final_loss")!; + expect(loss.comparable).toBe(false); + expect(loss.band).toBeNull(); + expect(report.flags.some((f) => f.startsWith("Target surface changed"))).toBe(true); +}); + +test("reform validation deltas match by id and flag beyond-band moves", () => { + const report = run({ + a: cal("a", {}), + b: cal("b", {}), + reformA: reforms([{ id: "obbba_salt", name: "SALT limit", absRel: 0.05 }]), + reformB: reforms([{ id: "obbba_salt", name: "SALT limit", absRel: 0.2 }]), + }); + expect(report.reforms_available).toBe(true); + const row = report.reforms.find((r) => r.id === "obbba_salt")!; + expect(row.delta).toBeCloseTo(0.15, 6); + expect(row.band).toBe("beyond"); + expect(report.flags.some((f) => f.includes("SALT limit"))).toBe(true); +}); + +test("reforms_available is false when one side never published", () => { + const report = run({ + a: cal("a", {}), + b: cal("b", {}), + reformA: reforms([{ id: "x", name: "X", absRel: 0.05 }]), + reformB: null, + }); + expect(report.reforms_available).toBe(false); + expect(report.reforms).toHaveLength(0); +}); + +test("a coverage shrink is detected and flagged", () => { + const coverage = (covered: number, missing: number): SourceCoverage => + ({ + available: true, + summary: { + covered_aliases: covered, + missing_aliases: missing, + reviewed_excluded_aliases: 0, + }, + }) as unknown as SourceCoverage; + const report = run({ + a: cal("a", {}), + b: cal("b", {}), + coverageA: coverage(25, 0), + coverageB: coverage(20, 3), + }); + expect(report.coverage_delta?.shrank).toBe(true); + expect(report.flags.some((f) => f.startsWith("Source coverage shrank"))).toBe(true); +}); + +test("a gate that was passing and is now waived is flagged as a loss of assurance", () => { + const cert = (outcome: string): Certification => + ({ + release_id: "r", + gates: [ + { + key: "ecps_parity", + label: "eCPS parity", + outcome, + source: "build_manifest", + reviewed_exclusions: [], + stale_exclusions: [], + failures: [], + failure_count: 0, + enforced: true, + evidence_sha: null, + summary: null, + }, + ], + totals: { total: 1, passed: 0, failed: 0, skipped: 0, waived: 0, enforced: 1 }, + reviewed_exclusion_registers: [], + stale_exclusion_count: 0, + }) as unknown as Certification; + const report = run({ + a: cal("a", {}), + b: cal("b", {}), + certA: cert("passed"), + certB: cert("waived"), + }); + expect(report.gate_changes).toHaveLength(1); + expect(report.gate_changes[0].kind).toBe("waived"); + expect(report.flags.some((f) => f.includes("eCPS parity"))).toBe(true); +}); + +test("formatDeltaTable and deltaSlackPayload render the report", () => { + const report = run({ + a: cal("a", { final_loss: 0.03 }), + b: cal("b", { final_loss: 0.08 }), + }); + const table = formatDeltaTable(report); + expect(table).toContain("Calibration loss"); + expect(table).toContain("flags:"); + const payload = deltaSlackPayload(report, { dashboardUrl: "https://example.test/compare" }); + expect(payload.blocks.length).toBeGreaterThanOrEqual(2); + expect(payload.text).toContain("Populace"); +}); + +test("no flags when everything moves within band", () => { + const report = run({ + a: cal("a", { final_loss: 0.03, fraction_within_10pct: 0.88, n_nonzero: 100000 }), + b: cal("b", { final_loss: 0.031, fraction_within_10pct: 0.882, n_nonzero: 100500 }), + }); + expect(report.flags).toHaveLength(0); + expect(formatDeltaTable(report)).toContain("within its band"); +}); diff --git a/frontend/lib/populace/deltas.ts b/frontend/lib/populace/deltas.ts new file mode 100644 index 0000000..70c19e4 --- /dev/null +++ b/frontend/lib/populace/deltas.ts @@ -0,0 +1,639 @@ +// Release-delta computation — the consumer-facing "which of my numbers moved, +// by how much, and is that move beyond its declared band" (populace#366's +// question, computed from what is on main today). One pure function feeds three +// surfaces: the Slack alert, the /api/populace/deltas/latest feed, and the +// in-page "since you last looked" banner. +// +// Bands (per metric family) live in delta-bands.json so thresholds are +// configuration, not code. A move beyond its band is what turns the alert loud. + +import { + buildComparison, + loadPointerReleaseId, + loadRelease, + loadReleases, + type Calibration, + type PopulaceCountry, +} from "./latest-artifact"; +import { + loadReformValidation, + type ReformValidation, + type ReformValidationMissing, +} from "./reforms"; +import { buildCertification, type Certification, type SideGateInput } from "./certification"; +import { loadSourceCoverage, type SourceCoverage, type CoverageMissing } from "./coverage"; +import bandConfig from "./delta-bands.json"; + +export interface MetricBand { + family: string; + abs?: number; + rel?: number; +} +export type BandConfig = Record; + +// The JSON carries a leading _comment key for maintainers; drop it. +export const DELTA_BANDS: BandConfig = Object.fromEntries( + Object.entries(bandConfig as Record).filter( + ([key, value]) => key !== "_comment" && typeof value === "object" && value != null, + ), +) as BandConfig; + +export type BandVerdict = "within" | "beyond" | null; +export type MetricUnit = "loss" | "share" | "count"; +export type ImproveDirection = "better" | "worse" | "flat" | null; + +export interface MetricDelta { + key: string; + label: string; + family: string | null; + unit: MetricUnit; + a: number | null; + b: number | null; + abs_delta: number | null; + rel_delta: number | null; + improve: ImproveDirection; + band: BandVerdict; + threshold: number | null; + comparable: boolean; + note: string | null; +} + +export interface ReformDelta { + id: string; + name: string; + category: string | null; + in_sample: boolean; + a_abs_rel: number | null; + b_abs_rel: number | null; + delta: number | null; // b − a change in |error|; negative = improving + band: BandVerdict; +} + +export interface CoverageDelta { + a_covered: number; + b_covered: number; + a_missing: number; + b_missing: number; + a_reviewed_excluded: number; + b_reviewed_excluded: number; + shrank: boolean; +} + +export interface GateChange { + key: string; + label: string; + a_outcome: string; + b_outcome: string; + kind: "waived" | "skipped" | "regressed"; +} + +export interface DeltaReport { + available: true; + a_release: string; + b_release: string; + a_date: string; + b_date: string; + generated_at: string; + headline: MetricDelta[]; + reforms: ReformDelta[]; + reforms_available: boolean; + surfaces_differ: boolean; + surface: { added: number; removed: number; improved: number; regressed: number }; + coverage_delta: CoverageDelta | null; + gate_changes: GateChange[]; + flags: string[]; + top_movers: MetricDelta[]; + bands: BandConfig; +} + +interface HeadlineSpec { + key: string; + label: string; + unit: MetricUnit; + improveIsLower: boolean | null; + needsComparable?: boolean; + get: (cal: Calibration) => number | null; +} + +const HEADLINE_SPECS: HeadlineSpec[] = [ + { + key: "final_loss", + label: "Calibration loss", + unit: "loss", + improveIsLower: true, + needsComparable: true, + get: (cal) => cal.final_loss, + }, + { + key: "fraction_within_10pct", + label: "Within 10% of target", + unit: "share", + improveIsLower: false, + get: (cal) => cal.fraction_within_10pct, + }, + { + key: "n_nonzero", + label: "Records kept", + unit: "count", + improveIsLower: null, + get: (cal) => cal.n_nonzero, + }, + { + key: "included_target_count", + label: "Targets included", + unit: "count", + improveIsLower: null, + get: (cal) => cal.included_target_count, + }, +]; + +const EPSILON = 1e-9; + +function relDelta(a: number | null, b: number | null): number | null { + if (a == null || b == null) return null; + if (Math.abs(a) < EPSILON) return null; + return (b - a) / Math.abs(a); +} + +function bandVerdict( + key: string, + absDelta: number | null, + rel: number | null, + bands: BandConfig, +): { band: BandVerdict; threshold: number | null } { + const band = bands[key]; + if (!band || absDelta == null) return { band: null, threshold: null }; + const absBeyond = band.abs != null && Math.abs(absDelta) > band.abs; + const relBeyond = band.rel != null && rel != null && Math.abs(rel) > band.rel; + const threshold = band.abs ?? band.rel ?? null; + return { band: absBeyond || relBeyond ? "beyond" : "within", threshold }; +} + +function improveOf(spec: HeadlineSpec, absDelta: number | null): ImproveDirection { + if (absDelta == null) return null; + if (spec.improveIsLower == null) return null; + if (Math.abs(absDelta) < EPSILON) return "flat"; + const lower = absDelta < 0; + return spec.improveIsLower === lower ? "better" : "worse"; +} + +function reformRows( + validation: ReformValidation | ReformValidationMissing | null | undefined, +): Map { + const map = new Map< + string, + { name: string; category: string | null; in_sample: boolean; absRel: number | null } + >(); + if (!validation || !validation.available) return map; + for (const row of validation.rows) { + map.set(row.id, { + name: row.name, + category: row.category, + in_sample: row.in_sample, + absRel: row.abs_relative_error, + }); + } + return map; +} + +function computeReformDeltas( + reformA: ReformValidation | ReformValidationMissing | null | undefined, + reformB: ReformValidation | ReformValidationMissing | null | undefined, + bands: BandConfig, +): ReformDelta[] { + const a = reformRows(reformA); + const b = reformRows(reformB); + const band = bands.reform_validation; + const rows: ReformDelta[] = []; + for (const [id, br] of b) { + const ar = a.get(id); + const delta = + ar?.absRel != null && br.absRel != null ? br.absRel - ar.absRel : null; + const verdict: BandVerdict = + delta == null || !band || band.abs == null + ? null + : Math.abs(delta) > band.abs + ? "beyond" + : "within"; + rows.push({ + id, + name: br.name, + category: br.category, + in_sample: br.in_sample, + a_abs_rel: ar?.absRel ?? null, + b_abs_rel: br.absRel, + delta, + band: verdict, + }); + } + // Worst regressions first (largest positive delta), then largest movers. + return rows.sort((x, y) => (y.delta ?? -Infinity) - (x.delta ?? -Infinity)); +} + +function coverageOf( + cov: SourceCoverage | CoverageMissing | null | undefined, +): { covered: number; missing: number; reviewed: number } | null { + if (!cov || !cov.available) return null; + return { + covered: cov.summary.covered_aliases, + missing: cov.summary.missing_aliases, + reviewed: cov.summary.reviewed_excluded_aliases, + }; +} + +function computeCoverageDelta( + coverageA: SourceCoverage | CoverageMissing | null | undefined, + coverageB: SourceCoverage | CoverageMissing | null | undefined, +): CoverageDelta | null { + const a = coverageOf(coverageA); + const b = coverageOf(coverageB); + if (!a || !b) return null; + return { + a_covered: a.covered, + b_covered: b.covered, + a_missing: a.missing, + b_missing: b.missing, + a_reviewed_excluded: a.reviewed, + b_reviewed_excluded: b.reviewed, + shrank: b.covered < a.covered || b.missing > a.missing, + }; +} + +function computeGateChanges( + certA: Certification | null | undefined, + certB: Certification | null | undefined, +): GateChange[] { + if (!certA || !certB) return []; + const aByKey = new Map(certA.gates.map((g) => [`${g.source}:${g.key}`, g])); + const changes: GateChange[] = []; + for (const gate of certB.gates) { + const prior = aByKey.get(`${gate.source}:${gate.key}`); + if (!prior) continue; + if (prior.outcome === gate.outcome) continue; + // Only surface losses of assurance: a gate that used to pass and is now + // waived, skipped, or failing. + if (prior.outcome === "passed" && gate.outcome === "waived") { + changes.push({ key: gate.key, label: gate.label, a_outcome: prior.outcome, b_outcome: gate.outcome, kind: "waived" }); + } else if (prior.outcome === "passed" && gate.outcome === "skipped") { + changes.push({ key: gate.key, label: gate.label, a_outcome: prior.outcome, b_outcome: gate.outcome, kind: "skipped" }); + } else if (prior.outcome === "passed" && gate.outcome === "failed") { + changes.push({ key: gate.key, label: gate.label, a_outcome: prior.outcome, b_outcome: gate.outcome, kind: "regressed" }); + } + } + return changes; +} + +function moverMagnitude(metric: MetricDelta): number { + if (metric.unit === "count") return Math.abs(metric.rel_delta ?? 0); + return Math.abs(metric.abs_delta ?? 0); +} + +export interface ComputeDeltaInput { + a: Calibration; + b: Calibration; + reformA?: ReformValidation | ReformValidationMissing | null; + reformB?: ReformValidation | ReformValidationMissing | null; + certA?: Certification | null; + certB?: Certification | null; + coverageA?: SourceCoverage | CoverageMissing | null; + coverageB?: SourceCoverage | CoverageMissing | null; + bands?: BandConfig; + now?: Date; +} + +function releaseDate(id: string): string { + return id.match(/(\d{8}(?:T\d{6}Z)?)$/)?.[1] ?? ""; +} + +export function computeReleaseDelta(input: ComputeDeltaInput): DeltaReport { + const { a, b } = input; + const bands = input.bands ?? DELTA_BANDS; + const comparison = buildComparison(a, b); + const surfacesDiffer = + a.rows.length !== b.rows.length || + comparison.summary.added > 0 || + comparison.summary.removed > 0; + const lossesComparable = comparison.summary.losses_comparable; + + const headline: MetricDelta[] = HEADLINE_SPECS.map((spec) => { + const av = spec.get(a); + const bv = spec.get(b); + const absDelta = av != null && bv != null ? bv - av : null; + const rel = relDelta(av, bv); + const comparable = spec.needsComparable ? lossesComparable : true; + const verdict = comparable + ? bandVerdict(spec.key, absDelta, rel, bands) + : { band: null as BandVerdict, threshold: bands[spec.key]?.abs ?? null }; + return { + key: spec.key, + label: spec.label, + family: bands[spec.key]?.family ?? null, + unit: spec.unit, + a: av, + b: bv, + abs_delta: absDelta, + rel_delta: rel, + improve: improveOf(spec, absDelta), + band: verdict.band, + threshold: verdict.threshold, + comparable, + note: + spec.needsComparable && !comparable + ? "Loss is not directly comparable across releases that calibrate to different target surfaces." + : null, + }; + }); + + const reforms = computeReformDeltas(input.reformA, input.reformB, bands); + const reformsAvailable = + !!input.reformA && input.reformA.available && !!input.reformB && input.reformB.available; + const coverageDelta = computeCoverageDelta(input.coverageA, input.coverageB); + const gateChanges = computeGateChanges(input.certA, input.certB); + + const flags: string[] = []; + for (const metric of headline) { + if (metric.band === "beyond") { + flags.push( + `${metric.label} moved ${fmtMetric(metric.a, metric.unit)} → ${fmtMetric(metric.b, metric.unit)} (Δ ${fmtDelta(metric)}), beyond its ${fmtThreshold(metric)} band.`, + ); + } + } + if (surfacesDiffer) { + flags.push( + `Target surface changed: +${comparison.summary.added} added, −${comparison.summary.removed} removed — loss not directly comparable.`, + ); + } + if (coverageDelta?.shrank) { + flags.push( + `Source coverage shrank: covered ${coverageDelta.a_covered} → ${coverageDelta.b_covered}, missing ${coverageDelta.a_missing} → ${coverageDelta.b_missing}.`, + ); + } + for (const change of gateChanges) { + flags.push(`Gate "${change.label}" ${change.a_outcome} → ${change.b_outcome}.`); + } + for (const reform of reforms) { + if (reform.band === "beyond" && reform.delta != null) { + flags.push( + `Reform "${reform.name}" |error| ${fmtPct(reform.a_abs_rel)} → ${fmtPct(reform.b_abs_rel)} (Δ ${reform.delta > 0 ? "+" : ""}${fmtPct(reform.delta)}).`, + ); + } + } + + const topMovers = [...headline] + .filter((m) => m.abs_delta != null && Math.abs(m.abs_delta) > EPSILON) + .sort((x, y) => { + const beyond = Number(y.band === "beyond") - Number(x.band === "beyond"); + return beyond || moverMagnitude(y) - moverMagnitude(x); + }); + + return { + available: true, + a_release: a.release_id, + b_release: b.release_id, + a_date: releaseDate(a.release_id), + b_date: releaseDate(b.release_id), + generated_at: (input.now ?? new Date()).toISOString(), + headline, + reforms, + reforms_available: reformsAvailable, + surfaces_differ: surfacesDiffer, + surface: { + added: comparison.summary.added, + removed: comparison.summary.removed, + improved: comparison.summary.improved, + regressed: comparison.summary.regressed, + }, + coverage_delta: coverageDelta, + gate_changes: gateChanges, + flags, + top_movers: topMovers, + bands, + }; +} + +// --- formatting (pure, dependency-free for the CLI/Slack path) -------------- + +function fmtPct(value: number | null | undefined, digits = 1): string { + if (value == null || !Number.isFinite(value)) return "—"; + return `${(value * 100).toFixed(digits)}%`; +} + +function fmtCount(value: number | null | undefined): string { + if (value == null || !Number.isFinite(value)) return "—"; + const abs = Math.abs(value); + const sign = value < 0 ? "-" : ""; + if (abs >= 1e9) return `${sign}${(abs / 1e9).toFixed(2)}B`; + if (abs >= 1e6) return `${sign}${(abs / 1e6).toFixed(2)}M`; + if (abs >= 1e3) return `${sign}${(abs / 1e3).toFixed(1)}K`; + return `${sign}${abs.toFixed(0)}`; +} + +function fmtLoss(value: number | null | undefined): string { + if (value == null || !Number.isFinite(value)) return "—"; + if (value === 0) return "0"; + if (Math.abs(value) < 1) return value.toFixed(value < 0.001 ? 6 : 4); + return value.toExponential(2).replace("e+", "e"); +} + +function fmtMetric(value: number | null, unit: MetricUnit): string { + if (unit === "share") return fmtPct(value); + if (unit === "count") return fmtCount(value); + return fmtLoss(value); +} + +function fmtDelta(metric: MetricDelta): string { + if (metric.abs_delta == null) return "—"; + const sign = metric.abs_delta > 0 ? "+" : ""; + if (metric.unit === "share") return `${sign}${(metric.abs_delta * 100).toFixed(1)}pp`; + if (metric.unit === "count") { + return `${sign}${fmtCount(metric.abs_delta)}${metric.rel_delta != null ? ` (${sign}${(metric.rel_delta * 100).toFixed(1)}%)` : ""}`; + } + return `${sign}${fmtLoss(metric.abs_delta)}`; +} + +function fmtThreshold(metric: MetricDelta): string { + if (metric.threshold == null) return "—"; + if (metric.unit === "share") return `${(metric.threshold * 100).toFixed(0)}pp`; + if (metric.unit === "count") return `${(metric.threshold * 100).toFixed(0)}%`; + return String(metric.threshold); +} + +function padEnd(value: string, width: number): string { + return value.length >= width ? value : value + " ".repeat(width - value.length); +} +function padStart(value: string, width: number): string { + return value.length >= width ? value : " ".repeat(width - value.length) + value; +} + +// A monospace text table for the CLI and the Slack code block. +export function formatDeltaTable(report: DeltaReport): string { + const lines: string[] = []; + const bandMark = (band: BandVerdict) => (band === "beyond" ? " ⚠ beyond" : band === "within" ? " ok" : ""); + const cols = [22, 12, 12, 16]; + lines.push( + padEnd("metric", cols[0]) + + padStart("previous", cols[1]) + + padStart("latest", cols[2]) + + padStart("Δ", cols[3]), + ); + lines.push("-".repeat(cols.reduce((s, c) => s + c, 0) + 10)); + for (const metric of report.headline) { + lines.push( + padEnd(metric.label, cols[0]) + + padStart(fmtMetric(metric.a, metric.unit), cols[1]) + + padStart(fmtMetric(metric.b, metric.unit), cols[2]) + + padStart(fmtDelta(metric), cols[3]) + + (metric.comparable ? bandMark(metric.band) : " (not comparable)"), + ); + } + if (report.reforms_available && report.reforms.length) { + lines.push(""); + lines.push("reform validation (|error| change, − = improving):"); + for (const reform of report.reforms.slice(0, 8)) { + lines.push( + padEnd(` ${reform.name}`, cols[0] + 2) + + padStart(fmtPct(reform.a_abs_rel), cols[1]) + + padStart(fmtPct(reform.b_abs_rel), cols[2]) + + padStart(reform.delta == null ? "—" : `${reform.delta > 0 ? "+" : ""}${fmtPct(reform.delta)}`, cols[3]) + + bandMark(reform.band), + ); + } + } + if (report.flags.length) { + lines.push(""); + lines.push("flags:"); + for (const flag of report.flags) lines.push(` • ${flag}`); + } else { + lines.push(""); + lines.push("flags: none — every tracked metric moved within its band."); + } + return lines.join("\n"); +} + +// Slack incoming-webhook payload (text + blocks). The dashboard URL deep-links +// to the compare view for the two releases. +export function deltaSlackPayload( + report: DeltaReport, + opts: { dashboardUrl?: string; country?: string } = {}, +): { text: string; blocks: unknown[] } { + const loud = report.flags.length > 0; + const country = (opts.country ?? "us").toUpperCase(); + const heading = loud + ? `:warning: Populace ${country} release delta — ${report.flags.length} flag${report.flags.length === 1 ? "" : "s"}` + : `:bar_chart: Populace ${country} release delta — all within band`; + const contextLine = [ + `\`${report.a_release}\``, + "→", + `\`${report.b_release}\``, + ].join(" "); + const blocks: unknown[] = [ + { type: "section", text: { type: "mrkdwn", text: `*${heading}*\n${contextLine}` } }, + { type: "section", text: { type: "mrkdwn", text: "```" + formatDeltaTable(report) + "```" } }, + ]; + if (opts.dashboardUrl) { + blocks.push({ + type: "context", + elements: [ + { + type: "mrkdwn", + text: `<${opts.dashboardUrl}?a=${encodeURIComponent(report.a_release)}&b=${encodeURIComponent(report.b_release)}|Compare in the calibration dashboard>`, + }, + ], + }); + } + return { text: heading, blocks }; +} + +// --- report loaders (network) ----------------------------------------------- +// One place that assembles a full DeltaReport for a pair of releases, shared by +// the /deltas/latest feed, the HF webhook alert, and the CLI script. Reform +// validation and source coverage are best-effort: a release that never +// published them just drops those sections. + +function sideGatesFor( + coverage: SourceCoverage | CoverageMissing | null, +): SideGateInput[] { + return [ + { + key: "us_source_coverage", + available: !!coverage && coverage.available, + gate: coverage && coverage.available ? coverage.gate : null, + enforced: + coverage && coverage.available ? coverage.classification === "release_gate" : null, + reviewed_exclusions: coverage && coverage.available ? coverage.reviewed_exclusions : [], + }, + // Kept present-but-unpublished on both sides so the gate sets align and a + // spurious "gate skipped" change is never emitted for them. + { key: "input_coverage", available: false }, + { key: "reform_coverage_smoke", available: false }, + ]; +} + +export interface DeltaUnavailable { + available: false; + reason: string; + a_release: string | null; + b_release: string | null; +} + +export async function loadReleaseDelta( + aId: string, + bId: string, + revalidate: number, + country: PopulaceCountry = "us", +): Promise { + const [a, b] = await Promise.all([ + loadRelease(aId, revalidate, country), + loadRelease(bId, revalidate, country), + ]); + // Reform validation is US-only (the JCT suite); skip it for UK. + const wantReforms = country === "us"; + const [reformA, reformB, coverageA, coverageB] = await Promise.all([ + wantReforms ? loadReformValidation(a.release_id, revalidate).catch(() => null) : null, + wantReforms ? loadReformValidation(b.release_id, revalidate).catch(() => null) : null, + loadSourceCoverage(a.release_id, revalidate, country).catch(() => null), + loadSourceCoverage(b.release_id, revalidate, country).catch(() => null), + ]); + const certA = buildCertification(a.build_manifest, a.release_id, sideGatesFor(coverageA)); + const certB = buildCertification(b.build_manifest, b.release_id, sideGatesFor(coverageB)); + return computeReleaseDelta({ + a, + b, + reformA, + reformB, + certA, + certB, + coverageA, + coverageB, + }); +} + +// Default target of the alert: the pointer's latest release vs the one before +// it in the registry. Returns an unavailable marker (not a throw) when there +// aren't two calibrated releases to compare. +export async function loadLatestDelta( + revalidate: number, + country: PopulaceCountry = "us", +): Promise { + const [releases, pointer] = await Promise.all([ + loadReleases(revalidate, country), + loadPointerReleaseId(revalidate, country).catch(() => ({ release_id: "", updated_at: null })), + ]); + const calibrated = releases.filter((r) => r.has_calibration); + if (calibrated.length < 2) { + return { + available: false, + reason: "Need at least two calibrated releases to compute a delta.", + a_release: calibrated[0]?.release_id ?? null, + b_release: null, + }; + } + // Prefer the pointer's latest; fall back to the newest by date. + const latestIndex = Math.max( + calibrated.findIndex((r) => r.release_id === pointer.release_id), + 0, + ); + const b = calibrated[latestIndex]; + const a = calibrated[latestIndex + 1] ?? calibrated[latestIndex === 0 ? 1 : 0]; + return loadReleaseDelta(a.release_id, b.release_id, revalidate, country); +} diff --git a/frontend/lib/populace/issue-links.test.ts b/frontend/lib/populace/issue-links.test.ts new file mode 100644 index 0000000..388a49c --- /dev/null +++ b/frontend/lib/populace/issue-links.test.ts @@ -0,0 +1,64 @@ +import { expect, test } from "bun:test"; + +import { extractIssueRefs, tokenizeIssueRefs, type IssueRef } from "./issue-links"; + +function refs(text: string): IssueRef[] { + return tokenizeIssueRefs(text).filter((t): t is IssueRef => typeof t !== "string"); +} + +test('"Issue #40" resolves to the default populace repo', () => { + const [ref] = refs("Reviewed exclusion: this release recalibrates the Issue #40 fiscal surface."); + expect(ref.owner).toBe("PolicyEngine"); + expect(ref.repo).toBe("populace"); + expect(ref.number).toBe(40); + expect(ref.url).toBe("https://github.com/PolicyEngine/populace/issues/40"); +}); + +test('"populace#359" resolves via the shortname map', () => { + const [ref] = refs("Sparse frozen-support cells, see populace#359."); + expect(ref.owner).toBe("PolicyEngine"); + expect(ref.repo).toBe("populace"); + expect(ref.number).toBe(359); + expect(ref.label).toBe("populace#359"); +}); + +test('bare "#359" resolves to the default repo', () => { + const [ref] = refs("tracked in #359"); + expect(ref.url).toBe("https://github.com/PolicyEngine/populace/issues/359"); +}); + +test("owner/repo#N resolves to that exact repo", () => { + const [ref] = refs("see PolicyEngine/policyengine-us#1234 for the variable"); + expect(ref.owner).toBe("PolicyEngine"); + expect(ref.repo).toBe("policyengine-us"); + expect(ref.number).toBe(1234); +}); + +test("known shortname maps to the right repo", () => { + const [ref] = refs("blocked on policyengine.py#462"); + expect(ref.repo).toBe("policyengine.py"); + expect(ref.number).toBe(462); +}); + +test("tokenize preserves surrounding text in order", () => { + const tokens = tokenizeIssueRefs("before #40 after"); + expect(tokens.length).toBe(3); + expect(tokens[0]).toBe("before "); + expect(typeof tokens[1]).not.toBe("string"); + expect(tokens[2]).toBe(" after"); +}); + +test("text with no issue reference yields a single string token", () => { + const tokens = tokenizeIssueRefs("no references here"); + expect(tokens).toEqual(["no references here"]); +}); + +test("extractIssueRefs de-duplicates by URL and keeps first-seen order", () => { + const list = extractIssueRefs("first #40 then populace#359 then #40 again"); + expect(list.map((r) => r.number)).toEqual([40, 359]); +}); + +test("empty / whitespace input yields no refs", () => { + expect(tokenizeIssueRefs("")).toEqual([]); + expect(extractIssueRefs(" ")).toEqual([]); +}); diff --git a/frontend/lib/populace/issue-links.ts b/frontend/lib/populace/issue-links.ts new file mode 100644 index 0000000..33bb13e --- /dev/null +++ b/frontend/lib/populace/issue-links.ts @@ -0,0 +1,124 @@ +// Reviewed-exclusion and gate reasons carry their tracking issue inline as free +// text — "Reviewed fiscal-refresh exclusion: … Issue #40 …", "populace#359", +// or a bare "#359". The #286 "cannot rot" contract requires every exclusion to +// name a live issue, so the coverage board and certification panel surface that +// link. This module turns those inline references into resolved GitHub URLs +// without a network call. + +export interface IssueRef { + // The text exactly as it appeared, e.g. "populace#359" or "Issue #40". + label: string; + owner: string; + repo: string; + number: number; + url: string; +} + +// Bare "#NNN" / "Issue #NNN" resolve here. The populace release artifacts are +// produced by PolicyEngine/populace, so its issue tracker is the default home +// for an unqualified reference. +export const DEFAULT_ISSUE_OWNER = "PolicyEngine"; +export const DEFAULT_ISSUE_REPO = "populace"; + +// Short repo names that appear as "#NNN" in producer text, mapped to the +// PolicyEngine repo they name. Anything not listed falls back to the default +// repo but keeps its literal label, so an unknown shorthand still links +// somewhere sensible rather than silently dropping. +const REPO_SHORTNAMES: Record = { + populace: { owner: "PolicyEngine", repo: "populace" }, + "populace-us-data": { owner: "PolicyEngine", repo: "populace-us-data" }, + "policyengine-us-data": { owner: "PolicyEngine", repo: "policyengine-us-data" }, + "calibration-diagnostics": { owner: "PolicyEngine", repo: "calibration-diagnostics" }, + "policyengine-us": { owner: "PolicyEngine", repo: "policyengine-us" }, + "policyengine-uk": { owner: "PolicyEngine", repo: "policyengine-uk" }, + "policyengine.py": { owner: "PolicyEngine", repo: "policyengine.py" }, +}; + +function issueUrl(owner: string, repo: string, number: number): string { + return `https://github.com/${owner}/${repo}/issues/${number}`; +} + +function resolveRef( + qualifier: string | undefined, + numberText: string, + label: string, +): IssueRef | null { + const number = Number(numberText); + if (!Number.isInteger(number) || number <= 0) return null; + if (qualifier && qualifier.includes("/")) { + const [owner, repo] = qualifier.split("/"); + if (owner && repo) { + return { label, owner, repo, number, url: issueUrl(owner, repo, number) }; + } + } + if (qualifier) { + const known = REPO_SHORTNAMES[qualifier.toLowerCase()]; + const owner = known?.owner ?? DEFAULT_ISSUE_OWNER; + const repo = known?.repo ?? DEFAULT_ISSUE_REPO; + return { label, owner, repo, number, url: issueUrl(owner, repo, number) }; + } + return { + label, + owner: DEFAULT_ISSUE_OWNER, + repo: DEFAULT_ISSUE_REPO, + number, + url: issueUrl(DEFAULT_ISSUE_OWNER, DEFAULT_ISSUE_REPO, number), + }; +} + +// Ordered alternatives: owner/repo#N, shorthand#N, "Issue #N" phrase, bare #N. +// A single global regex so one linear pass tokenizes the text; group indices +// tell which alternative fired. +function tokenRegex(): RegExp { + return new RegExp( + [ + "([A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+)#(\\d+)", // 1,2 owner/repo#N + "([A-Za-z0-9_.][A-Za-z0-9_.-]*)#(\\d+)", // 3,4 shorthand#N + "[Ii]ssues?\\s+#(\\d+)", // 5 "Issue #N" / "Issues #N" + "#(\\d+)", // 6 bare #N + ].join("|"), + "g", + ); +} + +// Split free text into a stream of plain strings and resolved issue references, +// in order, so a renderer can print the reason with its issue link inline. Plain +// runs between references are returned verbatim (never dropped). +export function tokenizeIssueRefs(text: string): Array { + if (!text) return []; + const tokens: Array = []; + const re = tokenRegex(); + let lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = re.exec(text)) !== null) { + const [full, ownerRepo, ownerRepoNum, shorthand, shorthandNum, phraseNum, bareNum] = + match; + let ref: IssueRef | null = null; + if (ownerRepo && ownerRepoNum) ref = resolveRef(ownerRepo, ownerRepoNum, full); + else if (shorthand && shorthandNum) ref = resolveRef(shorthand, shorthandNum, full); + else if (phraseNum) ref = resolveRef(undefined, phraseNum, full); + else if (bareNum) ref = resolveRef(undefined, bareNum, full); + // A zero-width or invalid match can't advance the loop — guard against it. + if (re.lastIndex === match.index) re.lastIndex += 1; + if (!ref) continue; + if (match.index > lastIndex) tokens.push(text.slice(lastIndex, match.index)); + tokens.push(ref); + lastIndex = match.index + full.length; + } + if (lastIndex < text.length) tokens.push(text.slice(lastIndex)); + return tokens; +} + +// The distinct issue references in a block of text, de-duplicated by URL and in +// first-seen order — for a compact "linked issues" chip row. +export function extractIssueRefs(text: string): IssueRef[] { + const seen = new Set(); + const refs: IssueRef[] = []; + for (const token of tokenizeIssueRefs(text)) { + if (typeof token === "string") continue; + if (seen.has(token.url)) continue; + seen.add(token.url); + refs.push(token); + } + return refs; +} diff --git a/frontend/lib/slack.ts b/frontend/lib/slack.ts index 0b8593b..bbcf5c9 100644 --- a/frontend/lib/slack.ts +++ b/frontend/lib/slack.ts @@ -1,6 +1,8 @@ import type { PopulaceCountry } from "@/lib/populace/latest-artifact"; +import { deltaSlackPayload, type DeltaReport } from "@/lib/populace/deltas"; const DASHBOARD_URL = "https://populace.dev/calibration/dashboard/populace"; +const COMPARE_URL = "https://populace.dev/calibration/dashboard/populace/compare"; const COUNTRY_LABEL: Record = { us: "🇺🇸 US", @@ -60,3 +62,29 @@ export async function postReleaseAlert(opts: { } return true; } + +// Post a computed release-delta table to the country's Slack incoming webhook. +// No-op (returns false) when that channel's webhook env var is unset — the same +// SLACK_WEBHOOK_POPULACE_{US,UK} the populace publish CLI uses. +export async function postDeltaAlert(opts: { + country: PopulaceCountry; + report: DeltaReport; +}): Promise { + const webhookUrl = process.env[WEBHOOK_ENV[opts.country]]; + if (!webhookUrl) return false; + + const payload = deltaSlackPayload(opts.report, { + dashboardUrl: COMPARE_URL, + country: opts.country, + }); + const res = await fetch(webhookUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`Slack webhook ${res.status}: ${body.slice(0, 200)}`); + } + return true; +} diff --git a/frontend/package.json b/frontend/package.json index e8a15c3..a4cc6c2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,7 +7,9 @@ "dev": "next dev --turbopack", "build": "next build", "start": "next start", - "lint": "tsc --noEmit" + "lint": "tsc --noEmit", + "test": "bun test", + "delta-alert": "bun run scripts/populace-delta-alert.ts" }, "dependencies": { "@policyengine/ui-kit": "^0.9.0", diff --git a/frontend/scripts/populace-delta-alert.ts b/frontend/scripts/populace-delta-alert.ts new file mode 100644 index 0000000..5ad3cba --- /dev/null +++ b/frontend/scripts/populace-delta-alert.ts @@ -0,0 +1,116 @@ +#!/usr/bin/env bun +// Release-delta alert. Given two release ids (default: latest vs the previous +// registry release), computes the headline delta table and posts it to the +// populace Slack webhook. No-ops with a clear log line when the webhook env var +// is unset, so it is safe to run anywhere (cron, CI, locally). +// +// bun run scripts/populace-delta-alert.ts # latest vs previous +// bun run scripts/populace-delta-alert.ts --a --b +// bun run scripts/populace-delta-alert.ts --country us --dry-run +// +// Env: SLACK_WEBHOOK_POPULACE_US / _UK (the same vars the populace publish CLI +// uses). Unset → printed only, never posted. + +import { parseCountry, type PopulaceCountry } from "@/lib/populace/latest-artifact"; +import { + formatDeltaTable, + loadLatestDelta, + loadReleaseDelta, + type DeltaReport, +} from "@/lib/populace/deltas"; +import { postDeltaAlert } from "@/lib/slack"; + +interface Args { + a?: string; + b?: string; + country: PopulaceCountry; + dryRun: boolean; + failOnFlags: boolean; + quietWhenClean: boolean; + help: boolean; +} + +function parseArgv(argv: string[]): Args { + const args: Args = { + country: "us", + dryRun: false, + failOnFlags: false, + quietWhenClean: false, + help: false, + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--a") args.a = argv[++i]; + else if (arg === "--b") args.b = argv[++i]; + else if (arg === "--country") args.country = parseCountry(argv[++i]); + else if (arg === "--dry-run") args.dryRun = true; + else if (arg === "--fail-on-flags") args.failOnFlags = true; + else if (arg === "--quiet-when-clean") args.quietWhenClean = true; + else if (arg === "--help" || arg === "-h") args.help = true; + } + return args; +} + +const HELP = `populace-delta-alert — post the release delta table to Slack. + +Options: + --a Earlier release (default: previous registry release) + --b Later release (default: latest via latest.json) + --country Dataset (default: us) + --dry-run Compute and print, never post to Slack + --fail-on-flags Exit 2 when any metric moved beyond its band + --quiet-when-clean Skip posting to Slack when nothing moved beyond band + --help Show this help + +Env: SLACK_WEBHOOK_POPULACE_US / SLACK_WEBHOOK_POPULACE_UK (unset → printed only).`; + +async function main(): Promise { + const args = parseArgv(process.argv.slice(2)); + if (args.help) { + console.log(HELP); + return 0; + } + + let report: DeltaReport; + if (args.a && args.b) { + report = await loadReleaseDelta(args.a, args.b, 0, args.country); + } else { + const latest = await loadLatestDelta(0, args.country); + if (!latest.available) { + console.log(`No delta to report: ${latest.reason}`); + return 0; + } + report = latest; + } + + console.log(`\nPopulace ${args.country.toUpperCase()} release delta`); + console.log(` previous: ${report.a_release}`); + console.log(` latest: ${report.b_release}\n`); + console.log(formatDeltaTable(report)); + console.log(""); + + const envVar = args.country === "uk" ? "SLACK_WEBHOOK_POPULACE_UK" : "SLACK_WEBHOOK_POPULACE_US"; + if (args.quietWhenClean && report.flags.length === 0) { + console.log("No beyond-band moves since the previous release — not posting (--quiet-when-clean)."); + } else if (args.dryRun) { + console.log(`[dry-run] not posting to Slack (${envVar}).`); + } else if (!process.env[envVar]) { + console.log(`${envVar} is unset — printed only, nothing posted.`); + } else { + const posted = await postDeltaAlert({ country: args.country, report }); + console.log(posted ? `Posted to Slack (${envVar}).` : `${envVar} unset — nothing posted.`); + } + + if (args.failOnFlags && report.flags.length > 0) { + console.error(`\n${report.flags.length} beyond-band flag(s) — exiting non-zero (--fail-on-flags).`); + return 2; + } + return 0; +} + +main() + .then((code) => process.exit(code)) + .catch((error) => { + console.error("Delta alert failed:", error instanceof Error ? error.message : error); + process.exit(1); + });