Proactive diagnostics: coverage board, certification panel, delta alerts, badges, CI (#101)

.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+# Cancel superseded runs on the same ref.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  frontend:
+    name: typecheck · test · build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.11
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Typecheck
+        run: bun run lint
+
+      - name: Test
+        run: bun test
+
+      - name: Build
+        run: bun run build
+        env:
+          # A bare (root) build in CI keeps the check independent of the
+          # populace.dev mount path; production sets the real basePath.
+          NEXT_PUBLIC_BASE_PATH: ""

.github/workflows/populace-delta-alert.yml

@@ -0,0 +1,51 @@
+name: Populace delta alert
+
+# The HF webhook (app/api/hf-webhook) posts the delta the moment a release
+# publishes. This scheduled run is the safety net: a weekly check that posts to
+# Slack only when the latest release moved beyond a declared band vs the
+# previous one (--quiet-when-clean). Also runnable on demand for an arbitrary
+# pair. No-ops with a clear log line when the webhook secret is unset.
+
+on:
+  schedule:
+    - cron: "0 13 * * 1" # Mondays 13:00 UTC
+  workflow_dispatch:
+    inputs:
+      a:
+        description: "Earlier release id (default: previous registry release)"
+        required: false
+      b:
+        description: "Later release id (default: latest)"
+        required: false
+      country:
+        description: "Dataset (us or uk)"
+        required: false
+        default: us
+
+jobs:
+  alert:
+    name: compute and post release delta
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.11
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Compute and post delta
+        env:
+          SLACK_WEBHOOK_POPULACE_US: ${{ secrets.SLACK_WEBHOOK_POPULACE_US }}
+          SLACK_WEBHOOK_POPULACE_UK: ${{ secrets.SLACK_WEBHOOK_POPULACE_UK }}
+        run: |
+          bun run scripts/populace-delta-alert.ts \
+            --country "${{ github.event.inputs.country || 'us' }}" \
+            ${{ github.event.inputs.a && format('--a {0}', github.event.inputs.a) || '' }} \
+            ${{ github.event.inputs.b && format('--b {0}', github.event.inputs.b) || '' }} \
+            ${{ github.event_name == 'schedule' && '--quiet-when-clean' || '' }}

docs/proactive-evidence/README.md

@@ -0,0 +1,36 @@
+# Proactive-diagnostics evidence (#101)
+
+Verification artifacts for the "buildable now" slice of the proactive-diagnostics
+epic ([#101](https://github.com/PolicyEngine/calibration-diagnostics/issues/101)).
+All screenshots and outputs are from a local `next start` reading the **live**
+`policyengine/populace-us` release resolved through `latest.json`
+(`populace-us-2024-buildi-sparse-rmloss100-6e8e929-20260709T034135Z`).
+
+## Screenshots
+
+| File | View |
+|---|---|
+| `coverage-board.png` | New **Coverage** view — source coverage manifest (10 hard-target families, 25/41 aliases covered, 16 reviewed exclusions each linking its issue), with the input-column (populace#369) and reform-smoke (populace#368) sections in their graceful "not published for this release" state. |
+| `certification-panel.png` | New **Certification** view — 13 gates (10 build-manifest + 3 side files), per-gate outcome / enforcement / evidence sha, forward-compatible with populace#381's `passed\|failed\|skipped\|waived` schema. |
+| `overview-delta-banner.png` | Overview with the **"since you last looked"** banner (localStorage seeded to the prior release): 4 beyond-band flags, top-mover chips, and the flag bullets. |
+
+## Delta script (two real releases)
+
+`delta-alert-output.txt` — `bun run scripts/populace-delta-alert.ts` computing
+latest vs the previous registry release, with the Slack webhook **unset** (the
+no-op log path). It flags the +12.4pp within-10% jump, the −83% targets-included
+drop, the changed target surface, and a real 27→25 source-coverage shrink.
+
+## Badge endpoints
+
+`badge-endpoints.txt` — the three shields.io endpoints served live:
+`default-release → buildi`, `gates → 11/11 (brightgreen)`,
+`within10 → 88.9% (brightgreen)`.
+
+## Verification (exit codes)
+
+```
+bun run lint   (tsc --noEmit)   → exit 0
+bun test                        → exit 0   (80 pass, 0 fail, 10 files)
+bun run build  (next build)     → exit 0   (all new routes + pages emitted)
+```

docs/proactive-evidence/badge-endpoints.txt

@@ -0,0 +1,11 @@
+# Live badge endpoints (localhost next start, real buildi release)
+
+GET /api/populace/badge/default-release
+{"schemaVersion":1,"label":"populace-us default","message":"buildi","color":"blue","cacheSeconds":600}
+
+GET /api/populace/badge/gates
+{"schemaVersion":1,"label":"gates","message":"11/11","color":"brightgreen","cacheSeconds":600}
+
+GET /api/populace/badge/within10
+{"schemaVersion":1,"label":"within 10%","message":"88.9%","color":"brightgreen","cacheSeconds":600}
+

docs/proactive-evidence/delta-alert-output.txt

@@ -0,0 +1,19 @@
+
+Populace US release delta
+  previous: populace-us-2024-sparse-l0-refit-57k-71a0887-national-only-20260701
+  latest:   populace-us-2024-buildi-sparse-rmloss100-6e8e929-20260709T034135Z
+
+metric                    previous      latest               Δ
+------------------------------------------------------------------------
+Calibration loss            0.0440      0.0308       -0.013180  (not comparable)
+Within 10% of target         76.5%       88.9%         +12.4pp  ⚠ beyond
+Records kept                 57.2K       57.2K     +58 (+0.1%)  ok
+Targets included             32.6K        5.5K -27.1K (-83.1%)  ⚠ beyond
+
+flags:
+  • Within 10% of target moved 76.5% → 88.9% (Δ +12.4pp), beyond its 2pp band.
+  • Targets included moved 32.6K → 5.5K (Δ -27.1K (-83.1%)), beyond its 5% band.
+  • Target surface changed: +49 added, −27172 removed — loss not directly comparable.
+  • Source coverage shrank: covered 27 → 25, missing 0 → 0.
+
+SLACK_WEBHOOK_POPULACE_US is unset — printed only, nothing posted.

frontend/app/api/hf-webhook/route.ts

@@ -3,7 +3,8 @@ import { timingSafeEqual } from "node:crypto";
 import { NextResponse } from "next/server";
 
 import type { PopulaceCountry } from "@/lib/populace/latest-artifact";
-import { postReleaseAlert } from "@/lib/slack";
+import { loadLatestDelta } from "@/lib/populace/deltas";
+import { postDeltaAlert, postReleaseAlert } from "@/lib/slack";
 
 export const runtime = "nodejs";
 // Push endpoint — must run on every call, never served from cache.
@@ -88,5 +89,18 @@ export async function POST(request: Request) {
     }
   }
 
-  return NextResponse.json({ ok: true, country, alerted });
+  // Follow the release ping with the computed headline delta (latest vs the
+  // previous registry release) — the epic's "on new latest.json, post the delta
+  // table". Best-effort and read-fresh; a failure here never fails the webhook.
+  let deltaPosted = false;
+  try {
+    const report = await loadLatestDelta(0, country);
+    if (report.available) {
+      deltaPosted = await postDeltaAlert({ country, report });
+    }
+  } catch (error) {
+    console.error("Delta alert failed:", error);
+  }
+
+  return NextResponse.json({ ok: true, country, alerted, deltaPosted });
 }

frontend/app/api/populace/badge/[metric]/route.ts

@@ -0,0 +1,96 @@
+import { NextResponse } from "next/server";
+
+import {
+  asObject,
+  hfResolveUrl,
+  loadPointerReleaseId,
+  parseCountry,
+  type PopulaceCountry,
+} from "@/lib/populace/latest-artifact";
+import { loadSourceCoverage } from "@/lib/populace/coverage";
+import { buildCertification, type SideGateInput } from "@/lib/populace/certification";
+import {
+  defaultReleaseBadge,
+  gatesBadge,
+  isBadgeMetric,
+  within10Badge,
+  type Shield,
+} from "@/lib/populace/badges";
+
+export const revalidate = 600;
+export const runtime = "nodejs";
+export const maxDuration = 60;
+
+// Fetch just build_manifest.json (small) — never the multi-MB diagnostics — so a
+// badge stays cheap. The calibration gate carries fraction_within_10pct too.
+async function fetchBuildManifest(
+  releaseId: string,
+  country: PopulaceCountry,
+): Promise<Record<string, unknown>> {
+  const url = hfResolveUrl(`releases/${releaseId}/build_manifest.json`, country);
+  const res = await fetch(url, { next: { revalidate }, signal: AbortSignal.timeout(20_000) });
+  if (!res.ok) throw new Error(`HF fetch failed ${res.status}: ${url}`);
+  return asObject(await res.json());
+}
+
+const ERROR_SHIELD: Shield = {
+  schemaVersion: 1,
+  label: "populace",
+  message: "unavailable",
+  color: "lightgrey",
+  cacheSeconds: 120,
+};
+
+export async function GET(
+  request: Request,
+  context: { params: Promise<{ metric: string }> },
+) {
+  const { metric } = await context.params;
+  const country = parseCountry(new URL(request.url).searchParams.get("country"));
+  if (!isBadgeMetric(metric)) {
+    return NextResponse.json(
+      { ...ERROR_SHIELD, message: "unknown badge" },
+      { status: 404 },
+    );
+  }
+  try {
+    const { release_id: releaseId } = await loadPointerReleaseId(revalidate, country);
+
+    if (metric === "default-release") {
+      return NextResponse.json(defaultReleaseBadge(releaseId, country));
+    }
+
+    if (metric === "within10") {
+      const manifest = await fetchBuildManifest(releaseId, country);
+      const calibration = asObject(asObject(manifest.gates).calibration);
+      const fraction =
+        typeof calibration.fraction_within_10pct === "number"
+          ? calibration.fraction_within_10pct
+          : null;
+      return NextResponse.json(within10Badge(fraction));
+    }
+
+    // metric === "gates"
+    const [manifest, source] = await Promise.all([
+      fetchBuildManifest(releaseId, country),
+      loadSourceCoverage(releaseId, revalidate, country),
+    ]);
+    const sideGates: SideGateInput[] = [
+      {
+        key: "us_source_coverage",
+        available: source.available,
+        gate: source.available ? source.gate : null,
+        enforced: source.available ? source.classification === "release_gate" : null,
+      },
+      { key: "input_coverage", available: false },
+      { key: "reform_coverage_smoke", available: false },
+    ];
+    const cert = buildCertification(manifest, releaseId, sideGates);
+    const ran = cert.totals.total - cert.totals.skipped;
+    return NextResponse.json(gatesBadge(cert.totals.passed, ran, cert.totals.failed));
+  } catch {
+    // Return a valid shield (200) so a README badge degrades gracefully rather
+    // than rendering a broken image.
+    return NextResponse.json(ERROR_SHIELD);
+  }
+}

frontend/app/api/populace/certification/route.ts

@@ -0,0 +1,74 @@
+import { NextResponse } from "next/server";
+
+import {
+  classifyApiError,
+  hfResolveUrl,
+  loadRelease,
+  parseCountry,
+  scrub,
+} from "@/lib/populace/latest-artifact";
+import {
+  loadInputColumnCoverage,
+  loadReformSmoke,
+  loadSourceCoverage,
+} from "@/lib/populace/coverage";
+import { buildCertification, type SideGateInput } from "@/lib/populace/certification";
+
+export const revalidate = 300;
+export const runtime = "nodejs";
+export const maxDuration = 120;
+
+export async function GET(request: Request) {
+  const params = new URL(request.url).searchParams;
+  const release = params.get("release") ?? "latest";
+  const country = parseCountry(params.get("country"));
+  try {
+    const cal = await loadRelease(release, revalidate, country);
+    const releaseId = cal.release_id;
+    const [source, inputColumns, reformSmoke] = await Promise.all([
+      loadSourceCoverage(releaseId, revalidate, country),
+      loadInputColumnCoverage(releaseId, revalidate, country),
+      loadReformSmoke(releaseId, revalidate, country),
+    ]);
+
+    const sideGates: SideGateInput[] = [
+      {
+        key: "us_source_coverage",
+        available: source.available,
+        gate: source.available ? source.gate : null,
+        // classification "release_gate" means this gate blocks publication.
+        enforced: source.available ? source.classification === "release_gate" : null,
+        reviewed_exclusions: source.available ? source.reviewed_exclusions : [],
+      },
+      {
+        key: "input_coverage",
+        available: inputColumns.available,
+        gate: inputColumns.available ? inputColumns.gate : null,
+        enforced: inputColumns.available ? inputColumns.enforced : null,
+      },
+      {
+        key: "reform_coverage_smoke",
+        available: reformSmoke.available,
+        gate: reformSmoke.available ? reformSmoke.gate : null,
+        enforced: reformSmoke.available ? reformSmoke.enforced : null,
+      },
+    ];
+
+    const certification = buildCertification(cal.build_manifest, releaseId, sideGates);
+    const prefix = `releases/${releaseId}`;
+    return NextResponse.json(
+      scrub({
+        release_id: releaseId,
+        updated_at: cal.updated_at,
+        certification,
+        source_artifacts: [
+          { name: "build_manifest", path: `${prefix}/build_manifest.json`, url: hfResolveUrl(`${prefix}/build_manifest.json`, country) },
+          { name: "us_source_coverage", path: `${prefix}/us_source_coverage.json`, url: hfResolveUrl(`${prefix}/us_source_coverage.json`, country) },
+        ],
+      }),
+    );
+  } catch (error) {
+    const { status, body } = classifyApiError(error);
+    return NextResponse.json(body, { status });
+  }
+}