diff --git a/API_LAYER_SCOPE.md b/API_LAYER_SCOPE.md deleted file mode 100644 index 2799b2c..0000000 --- a/API_LAYER_SCOPE.md +++ /dev/null @@ -1,316 +0,0 @@ -# Calibration Diagnostics API Layer Scope - -## Goal - -Create a stable, scriptable API layer for calibration diagnostics so users can inspect datasets, runs, H5 bundles, targets, aggregates, errors, and comparisons without depending on UI-oriented endpoints or dashboard state. - -The API should make these facts explicit: - -- Which target universe and calibration metadata are being used. -- Which H5 bundle produced each PE aggregate. -- Whether a target was included in the calibration loss. -- Whether loss contribution is meaningful for the selected target set. -- Which cache or computation path produced the result. - -## Non-Goals - -- Do not replace the existing dashboard routes in the first pass. -- Do not infer calibration loss membership when the published artifacts do not prove it. -- Do not make state/district H5 files look like standalone datasets unless the response also names the parent target DB and diagnostics source. -- Do not expose household-level contributor detail for dataset-mode runs until sparse matrix artifacts are available. - -## API Shape - -All stable endpoints should live under `/api/v1`. Existing dashboard endpoints can continue to serve the frontend while we migrate. - -### Dataset And Run Discovery - -`GET /api/v1/datasets` - -Returns dataset definitions and their supported layout: - -```json -{ - "items": [ - { - "dataset_id": "us-data-production", - "label": "US Data - Production Enhanced CPS", - "repo_id": "PolicyEngine/policyengine-us-data", - "repo_type": "model", - "layout": "root", - "primary_h5": "enhanced_cps_2024.h5" - } - ] -} -``` - -`GET /api/v1/datasets/{dataset_id}/runs` - -Returns runs available for a dataset. Root production has a synthetic `main` run. - -### Bundle Discovery - -`GET /api/v1/datasets/{dataset_id}/runs/{run_id}/bundles` - -Returns H5 bundles available for a run, grouped by bundle type: - -```json -{ - "dataset_id": "us-data-production", - "run_id": "main", - "items": [ - { - "bundle": "states/CA.h5", - "kind": "state", - "geography_id": "06", - "geography_name": "California", - "target_count": 116, - "included_target_count": 0, - "cache_status": "not_computed" - } - ] -} -``` - -The endpoint should support filters: - -- `kind=national|state|district|city|primary` -- `state_fips=6` -- `include_target_counts=true` -- `include_cache_status=true` - -### Run Summary - -`GET /api/v1/datasets/{dataset_id}/runs/{run_id}/summary` - -Query parameters: - -- `bundle`: optional H5 bundle path. If omitted, use the dataset primary H5. -- `included`: `true`, `false`, or omitted. -- `geo_level`: optional. -- `state_fips`: optional. - -Response should separate coverage from fit quality: - -```json -{ - "dataset_id": "us-data-production", - "run_id": "main", - "bundle": "states/CA.h5", - "target_universe_count": 116, - "included_target_count": 0, - "computed_target_count": 116, - "loss_contribution_available": false, - "metrics": { - "median_abs_rel_error": 0.21, - "mean_abs_rel_error": 0.37, - "p95_abs_rel_error": 1.4, - "total_loss": null - }, - "provenance": { - "target_db": "policy_data.db", - "diagnostics": "calibration/logs/unified_diagnostics.csv", - "aggregate_source": "states/CA.h5", - "calibration_pattern_source": null - } -} -``` - -### Targets - -`GET /api/v1/datasets/{dataset_id}/runs/{run_id}/targets` - -Stable query parameters: - -- `bundle` -- `geo_level` -- `state_fips` -- `geographic_id` -- `variable` -- `source` -- `included` -- `min_abs_rel_error` -- `sort` -- `order` -- `limit` -- `offset` - -Each target row should distinguish the facts currently overloaded in the UI: - -```json -{ - "target_id": 25388, - "target_name": "state/tax_unit_partnership_s_corp_income/56/[...]", - "variable": "tax_unit_partnership_s_corp_income", - "geo_level": "state", - "geographic_id": "56", - "target_value": 2208928000.0, - "pe_aggregate": 8863508975.77, - "rel_error": 3.01258, - "abs_rel_error": 3.01258, - "included_in_loss": false, - "loss_contribution": null, - "computed_from_bundle": "states/WY.h5", - "target_value_source": "policy_data.db", - "included_source": "unified_diagnostics.csv", - "calibration_pattern_source": null, - "eval_note": null -} -``` - -Rules: - -- `loss_contribution` should be `null` when the target is not known to be in loss. -- `included_in_loss` should only be true when a diagnostics row or target config proves inclusion. -- `computed_from_bundle` should always be present when `pe_aggregate` is present. - -### Target Detail - -`GET /api/v1/datasets/{dataset_id}/runs/{run_id}/targets/{target_id}` - -Returns a target row plus: - -- constraints -- provenance fields -- related bundle -- evaluation note -- optional compare fields - -Matrix-only detail such as contributors should stay separate and return explicit `501` for dataset-mode runs without matrix artifacts. - -### Evaluation Job - -`POST /api/v1/evaluate` - -Request: - -```json -{ - "dataset_id": "us-data-production", - "run_id": "main", - "bundle": "states/CA.h5", - "filters": { - "geo_level": ["state"], - "state_fips": [6], - "included": null - }, - "limit": 5000 -} -``` - -Response: - -```json -{ - "status": "complete", - "cache_status": "computed", - "elapsed_ms": 18320, - "result": { - "target_count": 116, - "computed_target_count": 116, - "items_url": "/api/v1/datasets/us-data-production/runs/main/targets?bundle=states/CA.h5" - } -} -``` - -First implementation can be synchronous because current bundle evaluation is request/response. If large states or districts make this too slow, promote the same contract to an async job with `job_id`. - -### Compare - -`POST /api/v1/compare` - -Supports: - -- run vs run -- bundle vs bundle -- primary H5 vs state/district H5 - -The response should include target matching keys and only compare rows with compatible target definitions. - -## Implementation Plan - -## Implemented In This Branch - -This branch implements the Phase 1 read-only API plus a synchronous evaluation wrapper: - -- `GET /api/v1/datasets` -- `GET /api/v1/datasets/{dataset_id}/runs` -- `GET /api/v1/datasets/{dataset_id}/runs/{run_id}/bundles` -- `GET /api/v1/datasets/{dataset_id}/runs/{run_id}/summary` -- `GET /api/v1/datasets/{dataset_id}/runs/{run_id}/targets` -- `GET /api/v1/datasets/{dataset_id}/runs/{run_id}/targets/{target_id}` -- `POST /api/v1/evaluate` -- `POST /api/v1/compare` - -It also exposes the current top-level us-data staging snapshot as -`us-data-current-staging` / run `staging`, backed by -`staging/calibration/policy_data.db`, `staging/calibration/source_imputed_stratified_extended_cps.h5`, -and top-level `staging/{states,districts,national,cities}` H5 bundles. - -Not yet implemented: - -- async evaluation jobs -- generated notebook/CI client - -### Phase 1: Contract And Read-Only Wrappers - -- Add Pydantic response models under `backend/api/v1/models.py`. -- Add a new router mounted at `/api/v1`. -- Implement dataset, run, bundle, summary, and target list endpoints by wrapping existing services. -- Preserve existing dashboard routes unchanged. -- Add tests for schema stability and key semantics. - -### Phase 2: Evaluation Semantics - -- Make bundle evaluation metadata explicit: - - `computed_from_bundle` - - `cache_status` - - `eval_note` - - `loss_contribution_available` -- Return `loss_contribution: null` for targets not known to be in loss. -- Add bundle cache inspection helpers. - -### Phase 3: Compare And CI Use - -- Add compare endpoint. -- Add JSON examples for CI usage. -- Add a small CLI or documented `curl` recipes for common checks: - - latest run summary - - worst state targets - - compare current production state bundle to staging state bundle - -## Testing Strategy - -Backend tests: - -- Dataset/runs endpoint returns root, staging, and H5 metadata. -- Bundles endpoint lists `states/CA.h5` for `us-data-production/main`. -- Targets endpoint marks skipped state rows with `loss_contribution: null`. -- Targets endpoint returns `computed_from_bundle` when PE aggregate is present. -- Summary endpoint reports `loss_contribution_available=false` for state-only skipped rows. -- Compare endpoint rejects incompatible datasets with a clear error. - -Smoke tests: - -```bash -uv run --python 3.12 pytest -uv run --python 3.12 ruff check backend tests -cd frontend && bun run lint -``` - -API smoke examples: - -```bash -curl 'http://localhost:8000/api/v1/datasets' -curl 'http://localhost:8000/api/v1/datasets/us-data-production/runs/main/bundles?kind=state' -curl 'http://localhost:8000/api/v1/datasets/us-data-production/runs/main/targets?bundle=states/CA.h5&geo_level=state&included=false' -curl 'http://localhost:8000/api/v1/datasets/us-data-current-staging/runs/staging/bundles?kind=state' -``` - -## Open Questions - -- Should `included=false` mean explicitly excluded, or simply not proven included? -- Can `target_config.yaml` be published alongside root production artifacts so state/district calibration patterns are provable? -- Should state H5 files appear in the top dataset selector, or stay as bundle selectors under the parent dataset/run? -- Should large bundle evaluations become async jobs before exposing the stable API publicly? -- Do we want an OpenAPI client generated for notebooks and CI? diff --git a/ARTIFACTS.md b/ARTIFACTS.md deleted file mode 100644 index b0d5458..0000000 --- a/ARTIFACTS.md +++ /dev/null @@ -1,142 +0,0 @@ -# Calibration artifact publishing spec - -This document is the contract between the **PolicyEngine data pipeline** -(the producer of calibration runs) and **calibration-diagnostics** (the -dashboard that visualises them). - -## TL;DR for the data team - -To unlock the dashboard against canonical `PolicyEngine/policyengine-us-data` -runs, please add two files to each staging publish: - -| File | How to produce it | -|---|---| -| `X_sparse.npz` | Save the `X_sparse` your `UnifiedMatrixBuilder.build_matrix()` already returns. `scipy.sparse.save_npz(staging_dir / "X_sparse.npz", X_sparse)`. | -| `calibration_weights.npy` | The final per-household weights vector. `np.save(staging_dir / "calibration_weights.npy", final_weights)`. | - -Optional but very useful: - -| File | Why | -|---|---| -| `target_names.json` | The string labels for each target row in `X_sparse`. Cheap to dump (`json.dump(target_names, ...)`). | -| `target_config.yaml` | Source-of-truth for which targets the calibration is opted into. The dashboard surfaces the REMOVED commentary, so it's a high-leverage file. | - -Two-line change in the pipeline (post-`build_matrix`) is enough. Everything -else (policy_data.db, enhanced_cps_2024.h5) is already published. - -The rest of this document is the longer-form spec. - ---- - -## Why this is needed - -The dashboard's core diagnostics — error per target, loss contribution, -contributor analysis, convergence — all need the **household-by-target -sparse matrix** that the pipeline builds during calibration. The pipeline -needs it too (it's how the optimizer fits weights), so it's already being -computed; it just isn't published as an artifact. - -Without it, the dashboard can either: -- Rebuild X at load time — tried, takes 1h+ per run, unusable, OR -- Compute only the trivial geographic-only aggregates (~2% of targets), OR -- Ship the published artifact and load it in seconds (this proposal) - -## Concepts - -- **Dataset** — a logically-distinct calibration (e.g. US enhanced CPS, US - PUF, UK). One HuggingFace repo per dataset. -- **Run** — a single build of that calibration. One prefix per run in the - dataset's repo. - -Two prefix conventions are supported: - -| Layout | Pattern | Example | -|---|---|---| -| **flat** | `//` | `policyengine-us-data-pipeline/test/calibration_package.pkl` | -| **staging** | `/staging//` | `policyengine-us-data/staging/usdata-gha25719239158-a1-889ab438/policy_data.db` | - -The dashboard auto-detects which layout a dataset uses (declared in -`DatasetConfig.layout`). - -## Files per run - -### Required for full diagnostics - -| File | What | -|---|---| -| `X_sparse.npz` *or* `calibration_package.pkl` | The sparse household-by-target contribution matrix. `.npz` (scipy.sparse) is preferred for canonical staging; `.pkl` (the legacy `load_calibration_package` bundle, containing X plus metadata) is what the sandbox uses today. | -| `calibration_weights.npy` | Final per-household weights (1-D float array). | -| `policy_data.db` | SQLite DB with the canonical `targets`, `strata`, `stratum_constraints` tables. | -| `enhanced_cps_2024.h5` *(or other primary microdata h5)* | The calibrated dataset itself — needed to evaluate PE variables for comparison. | - -If `X_sparse.npz` is present, the dashboard computes `estimate = X @ weights` -directly. If only the four `.pkl`-bundle keys are present, same thing via -the legacy code path. - -### Optional (each unlocks a specific view) - -| File | What it enables | -|---|---| -| `target_names.json` | Cleaner labels in the Target Explorer (otherwise derived from variable/geo/constraint columns). | -| `target_config.yaml` | The Used / Unused split + REMOVED commentary surfaced in the dashboard. | -| `calibration_log.csv` | Per-target convergence traces (Target Explorer detail tab). | -| `unified_diagnostics.csv` | Per-category epoch summaries. | -| `source_imputed_stratified_extended_cps.h5` | Detail-panel views that need the pre-calibration source data. | - -## Path convention - -``` -/[staging/]/ -``` - -Where: -- `` — any HF model repo. e.g. - `PolicyEngine/policyengine-us-data`. -- `` — uniquely identifies the build. Either the team's own - convention (`usdata-gha25719239158-a1-889ab438`) or a versioned one - (`1.110.12_22f922eb_20260329T2233Z`). - -## Registering a new dataset in the dashboard - -After the data team publishes a new dataset's first run to HF, add an entry -to `DEFAULT_DATASETS` in `backend/services/runs.py`: - -```python -DatasetConfig( - id="us-puf", # short id, used in URLs - label="US PUF-enhanced", # shown in dropdown - repo_id="PolicyEngine/policyengine-us-puf-pipeline", - layout="staging", # or "flat" - primary_h5="enhanced_cps_2024.h5", # for staging layout -), -``` - -Runs within the repo are auto-discovered: any prefix whose contents -include the required files appears in the Run dropdown. - -## Minimum viable publish - -If you only have time to add two files to your existing staging output: - -1. `X_sparse.npz` (the matrix you already build internally) -2. `calibration_weights.npy` (the weights you already optimise) - -That's enough to unlock the dashboard's full diagnostics suite on every -existing staging run. The other files are nice-to-haves. - -## Example (canonical staging) - -``` -PolicyEngine/policyengine-us-data/ -└── staging/ - └── usdata-gha25719239158-a1-889ab438/ - ├── X_sparse.npz ← please add (required) - ├── calibration_weights.npy ← please add (required) - ├── target_names.json nice-to-have - ├── target_config.yaml nice-to-have - ├── policy_data.db ← already published - ├── enhanced_cps_2024.h5 ← already published - ├── cps_2024.h5 ← already published - ├── small_enhanced_cps_2024.h5 ← already published - └── _run_context.json ← already published -``` diff --git a/Makefile b/Makefile index 7649ce3..0b7b10d 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,16 @@ -.PHONY: frontend backend install-frontend install-backend +.PHONY: install dev build test typecheck -install-frontend: +install: cd frontend && bun install -install-backend: - pip install -e . +dev: + cd frontend && bun run dev -frontend: - cd frontend && NEXT_PUBLIC_USE_FIXTURES=true bun run dev +build: + cd frontend && bun run build -backend: - uvicorn backend.app:app --reload --port 8000 +typecheck: + cd frontend && bun run lint + +test: + cd frontend && bun test diff --git a/README.md b/README.md index 4460db6..e217ba7 100644 --- a/README.md +++ b/README.md @@ -1,85 +1,50 @@ -# Calibration Diagnostics - -Interactive tool for diagnosing calibration quality in PolicyEngine's Enhanced CPS dataset builds. Provides a FastAPI backend and Next.js frontend for exploring calibration artifacts — sparse matrices, target databases, calibration logs, and Microsimulation outputs. - -## What this does - -After the PolicyEngine data pipeline calibrates survey weights against ~3,000 demographic and economic targets, this tool helps answer: - -- **Which targets have the worst fit?** Sortable/filterable target table with error metrics and pull scores. -- **Why is a target badly fit?** Error decomposition (raw data vs initial weights vs final weights), contributor analysis, and automated constraint auditing against the targets database. -- **Which households are distorted?** Filter by g-weight and any variable to find households whose weights shifted most, inspect their full variable profiles, and see which targets are pulling their weights. -- **What shifted under calibration?** Decompose any composite variable (e.g., `spm_unit_net_income`) into its formula dependencies and see which components changed most. -- **How did calibration converge?** Per-target and per-category error traces over training epochs. - -## Architecture - -``` -backend/ Python FastAPI (20 endpoints) -frontend/ Next.js 15 + React 19 + @policyengine/ui-kit -``` - -The backend loads calibration artifacts at startup and exposes REST endpoints. The frontend calls these endpoints and provides interactive visualization. - -## Quick start - -### Frontend (fixture mode — no backend needed) - -```bash -cd frontend -bun install -NEXT_PUBLIC_USE_FIXTURES=true bun run dev -``` - -Open http://localhost:3000. All views render with sample data. - -### Backend - -Requires calibration artifacts (calibration_package.pkl, calibration_weights.npy, policy_data.db, CPS H5 dataset). - -```bash -pip install -e . -PACKAGE_PATH=/path/to/calibration_package.pkl \ -WEIGHTS_PATH=/path/to/calibration_weights.npy \ -DB_PATH=/path/to/policy_data.db \ -DATASET_PATH=/path/to/extended_cps_2024.h5 \ -uvicorn backend.app:app --reload -``` - -### Both via Make +# Populace calibration diagnostics + +Interactive dashboard for the **populace-US** synthetic population — PolicyEngine's +calibrated microdataset published on Hugging Face at +[`policyengine/populace-us`](https://huggingface.co/datasets/policyengine/populace-us). + +Everything is read **live from Hugging Face**: the current release is resolved +through `latest.json`, and each release's manifests and per-target calibration +diagnostics are fetched on demand. There is no committed data snapshot and no +separate backend — the Next.js API routes are the API layer. + +## What it shows + +- **Release summary** (`/populace`) — calibration loss and convergence, within-10% + and within-tolerance, records kept after L0, acceptance gates, solver + provenance, per-family fit, and worst-fit / biggest-improvement targets, for + the current release (or any release via `?release=`). +- **Target diagnostics** (`/populace/targets`) — browse the calibration target + surface by the quantity each constraint measures (e.g. *adjusted gross income*), + then drill its breakdown dimensions (income band x return type x filing status, + geography, ...). Every axis a variable varies on becomes a filterable, sortable + facet, and any target opens a canonical detail card (structured registry + fields, source citation, initial -> final -> target). +- **Compare versions** (`/populace/compare`) — diff two releases: targets matched + by name, common targets get a fit change, and added/removed targets are + surfaced. + +## API + +The Next.js route handlers are the API layer; all read live from Hugging Face: + +| Endpoint | Purpose | +|---|---| +| `GET /api/populace/releases` | List published releases (newest first) | +| `GET /api/populace?release=` | Release summary (default: latest) | +| `GET /api/populace/target-diagnostics?release=&...` | Faceted per-target diagnostics | +| `GET /api/populace/compare?a=&b=` | Version-over-version diff | + +## Develop ```bash -make install-frontend -make install-backend -make frontend # starts frontend on port 3000 -make backend # starts backend on port 8000 +make install # cd frontend && bun install +make dev # next dev (http://localhost:3000) +make typecheck # tsc --noEmit +make test # bun test (data-layer suite) +make build # next build ``` -## Frontend views - -| View | Description | -|------|-------------| -| Overview | Weight distribution stats, income distribution comparison, worst-fit targets | -| Target Explorer | Sortable target table with detail panel (error decomposition, provenance, constraint audit, contributors, convergence) | -| Weight Landscape | G-weight histogram with metric/slice controls, distribution stats | -| Variable Decomposition | Decompose any variable into shifted components | -| Household Inspector | Filter distorted households by any variable, inspect profiles and target attributions | -| Convergence | Per-category and per-target error traces over epochs | - -## Backend endpoints - -20 endpoints across 7 route groups: - -- `POST /decompose` — variable decomposition -- `GET /targets`, `/targets/search`, `/targets/poverty-impact`, `/targets/{id}/error-decomposition`, `/targets/{id}/provenance`, `/targets/{id}/eligibility-audit`, `/targets/{id}/constraint-diff`, `/targets/{id}/contributors`, `/targets/{id}/convergence` -- `GET /strata/{id}` -- `GET /households/distorted`, `/households/{id}/profile`, `/households/{id}/attributions` -- `GET /weights/distribution`, `/weights/histogram` -- `GET /statistics/poverty-rate`, `/statistics/income-distribution` -- `GET /epochs/summary`, `/epochs/traces` - -## Dependencies - -**Backend:** FastAPI, uvicorn, policyengine-us-data, policyengine-us, scipy, numpy, pandas, sqlalchemy, sqlmodel - -**Frontend:** Next.js 15, React 19, @policyengine/ui-kit, TanStack React Query v5, Tailwind CSS v4, Recharts +Optional env: `POPULACE_HF_REPO`, `POPULACE_HF_REVISION` to point at a different +dataset/revision; `NEXT_PUBLIC_API_URL` to call an external API base. diff --git a/SUPABASE_PERSISTENCE_SCOPE.md b/SUPABASE_PERSISTENCE_SCOPE.md deleted file mode 100644 index 5b253be..0000000 --- a/SUPABASE_PERSISTENCE_SCOPE.md +++ /dev/null @@ -1,232 +0,0 @@ -# Supabase Persistence Layer Scope - -## Goal - -Add a small persistence layer for shared, queryable diagnostics state without -moving large immutable artifacts out of Hugging Face. - -Supabase should make the dashboard and API more usable when evaluations become -async, shared across users, or deployed across multiple backend instances. - -## Current State - -The app does not currently store anything in Supabase. - -Current storage paths: - -- Hugging Face stores source artifacts: `policy_data.db`, H5 datasets, - diagnostics CSVs, and staging/root bundle trees. -- Local `.artifacts/` stores downloaded artifacts and computed pickle caches. -- Backend memory stores loaded runs and bundle simulations for one process. -- Downloaded `policy_data.db` is read as SQLite for target metadata. - -## Non-Goals - -- Do not store H5 files in Supabase. -- Do not store full `policy_data.db` copies in Supabase. -- Do not store sparse matrices, household-level matrices, or large result - frames in Postgres. -- Do not make Supabase the source of truth for immutable published artifacts. -- Do not require Supabase for local development. - -## What Belongs In Supabase - -### Evaluation Jobs - -Track synchronous-now, async-later API work: - -- `job_id` -- `status`: `queued`, `running`, `complete`, `failed`, `cancelled` -- `dataset_id` -- `run_id` -- `bundle` -- request filters as `jsonb` -- `created_at`, `started_at`, `finished_at` -- `elapsed_ms` -- `error_message` -- result metadata: target counts, computed counts, result URL or object key - -This is the first table to implement. - -### Bundle Evaluation Cache Metadata - -Track whether a bundle has already been evaluated and where results live: - -- `dataset_id` -- `run_id` -- `bundle` -- artifact revision or digest where available -- `target_count` -- `computed_target_count` -- `cache_status` -- `cache_key` -- `created_at`, `updated_at` - -The computed result body can remain in local `.artifacts` for dev. In deployed -environments, store result files in object storage and put only the object key -in Supabase. - -### Summary Snapshots - -Persist small aggregate summaries: - -- dataset/run/bundle/filter signature -- target universe count -- included target count -- computed target count -- median/mean/p95 absolute relative error -- total loss when meaningful -- provenance metadata - -These can power fast dashboard landing states and CI checks. - -### Saved Comparisons - -Optional after jobs exist: - -- comparison side A/B identifiers -- matched target count -- computed pair count -- improved/regressed counts -- top movers object key or compact JSON - -## Proposed Schema - -### `diagnostics_evaluation_jobs` - -```sql -create table diagnostics_evaluation_jobs ( - job_id uuid primary key default gen_random_uuid(), - status text not null check ( - status in ('queued', 'running', 'complete', 'failed', 'cancelled') - ), - dataset_id text not null, - run_id text not null, - bundle text, - filters jsonb not null default '{}'::jsonb, - request_hash text not null, - target_count integer, - computed_target_count integer, - result_url text, - result_object_key text, - error_message text, - elapsed_ms double precision, - created_at timestamptz not null default now(), - started_at timestamptz, - finished_at timestamptz -); - -create index diagnostics_evaluation_jobs_lookup - on diagnostics_evaluation_jobs (dataset_id, run_id, bundle, request_hash); - -create index diagnostics_evaluation_jobs_status - on diagnostics_evaluation_jobs (status, created_at desc); -``` - -### `diagnostics_bundle_cache` - -```sql -create table diagnostics_bundle_cache ( - dataset_id text not null, - run_id text not null, - bundle text not null, - artifact_revision text, - cache_key text not null, - cache_status text not null, - target_count integer, - computed_target_count integer, - result_object_key text, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now(), - primary key (dataset_id, run_id, bundle, cache_key) -); -``` - -### `diagnostics_summary_snapshots` - -```sql -create table diagnostics_summary_snapshots ( - summary_id uuid primary key default gen_random_uuid(), - dataset_id text not null, - run_id text not null, - bundle text not null, - filter_hash text not null, - filters jsonb not null default '{}'::jsonb, - target_universe_count integer not null, - included_target_count integer not null, - computed_target_count integer not null, - loss_contribution_available boolean not null, - metrics jsonb not null default '{}'::jsonb, - provenance jsonb not null default '{}'::jsonb, - created_at timestamptz not null default now() -); - -create unique index diagnostics_summary_snapshots_key - on diagnostics_summary_snapshots (dataset_id, run_id, bundle, filter_hash); -``` - -## API Changes - -### Phase 1: Optional Persistence Client - -- Add optional Supabase settings: - - `SUPABASE_URL` - - `SUPABASE_SERVICE_ROLE_KEY` - - `DIAGNOSTICS_PERSISTENCE=disabled|supabase` -- Add a backend persistence interface with a no-op implementation. -- Add Supabase implementation behind the interface. -- Local development defaults to no-op. - -### Phase 2: Evaluation Job Recording - -- Keep `POST /api/v1/evaluate` synchronous initially. -- Record one job row per request when Supabase is configured. -- Update the job row to `complete` or `failed`. -- Return `job_id` in the response when persistence is enabled. - -### Phase 3: Async Evaluation - -- Promote `POST /api/v1/evaluate` to optionally return `202 Accepted`. -- Add: - - `GET /api/v1/jobs/{job_id}` - - `GET /api/v1/jobs/{job_id}/result` -- Worker can be in-process first, then moved to a separate worker. - -### Phase 4: Shared Cache Metadata - -- Write bundle evaluation cache metadata after bundle evaluation completes. -- Use Supabase metadata to show cache status before loading a run. -- Keep the actual heavy result file outside Postgres. - -## Security - -- Backend uses service role key only server-side. -- Frontend never receives Supabase service credentials. -- Public read policies are not required for Phase 1 because the FastAPI layer - mediates access. -- If direct client reads are added later, expose only aggregate metadata and - saved views, not raw job errors or internal paths. - -## Open Questions - -- Which object store should hold deployed result files: Supabase Storage, - Vercel Blob, S3, or Hugging Face artifacts? -- Do we need user identity for saved comparisons, or are all saved objects - workspace-global? -- Should repeated identical evaluate requests reuse an existing complete job - by `request_hash`, or create a new audit row every time? -- What retention policy should apply to result objects and job rows? -- Should production disable local `.artifacts` writes entirely or keep them as - a node-local warm cache? - -## Recommended First Implementation - -Implement only Phase 1 and Phase 2 first: - -- Add no-op and Supabase persistence adapters. -- Add migrations for `diagnostics_evaluation_jobs`. -- Add `job_id` to `POST /api/v1/evaluate` when persistence is enabled. -- Keep all existing API behavior synchronous and backward compatible. - -This gives us durable observability without forcing async worker design or -large-object storage decisions yet. diff --git a/backend/__init__.py b/backend/__init__.py deleted file mode 100644 index 82432a8..0000000 --- a/backend/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Calibration diagnostics API package.""" diff --git a/backend/api/__init__.py b/backend/api/__init__.py deleted file mode 100644 index 1ddeb13..0000000 --- a/backend/api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Versioned public API package.""" diff --git a/backend/api/v1/__init__.py b/backend/api/v1/__init__.py deleted file mode 100644 index c4f50de..0000000 --- a/backend/api/v1/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Stable v1 diagnostics API.""" diff --git a/backend/api/v1/models.py b/backend/api/v1/models.py deleted file mode 100644 index 15181de..0000000 --- a/backend/api/v1/models.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Pydantic schemas for the stable diagnostics API.""" - -from pydantic import BaseModel, Field - - -class DatasetItem(BaseModel): - dataset_id: str - label: str - repo_id: str - repo_type: str - layout: str - primary_h5: str - - -class DatasetListResponse(BaseModel): - items: list[DatasetItem] - - -class RunItem(BaseModel): - dataset_id: str - run_id: str - label: str - last_modified: str | None = None - - -class RunListResponse(BaseModel): - dataset_id: str - items: list[RunItem] - - -class BundleItem(BaseModel): - bundle: str - kind: str - geography_id: str | None = None - geography_name: str | None = None - target_count: int | None = None - included_target_count: int | None = None - cache_status: str | None = None - - -class BundleListResponse(BaseModel): - dataset_id: str - run_id: str - items: list[BundleItem] - - -class ProvenanceInfo(BaseModel): - target_db: str - diagnostics: str | None = None - aggregate_source: str - calibration_pattern_source: str | None = None - - -class SummaryMetrics(BaseModel): - median_abs_rel_error: float | None = None - mean_abs_rel_error: float | None = None - p95_abs_rel_error: float | None = None - total_loss: float | None = None - - -class SummaryResponse(BaseModel): - dataset_id: str - run_id: str - bundle: str - target_universe_count: int - included_target_count: int - computed_target_count: int - loss_contribution_available: bool - metrics: SummaryMetrics - provenance: ProvenanceInfo - - -class TargetItem(BaseModel): - target_id: int | None = None - target_name: str - variable: str - geo_level: str | None = None - geographic_id: str | None = None - target_value: float | None = None - pe_aggregate: float | None = None - rel_error: float | None = None - abs_rel_error: float | None = None - included_in_loss: bool - loss_contribution: float | None = None - computed_from_bundle: str | None = None - target_value_source: str - included_source: str | None = None - calibration_pattern_source: str | None = None - eval_note: str | None = None - - -class TargetListResponse(BaseModel): - dataset_id: str - run_id: str - bundle: str - items: list[TargetItem] - total: int - offset: int - limit: int - - -class EvaluationFilters(BaseModel): - geo_level: list[str] | None = None - state_fips: list[int] | None = None - geographic_id: str | None = None - variable: list[str] | None = None - source: list[str] | None = None - included: bool | None = None - min_abs_rel_error: float | None = None - - -class EvaluationRequest(BaseModel): - dataset_id: str - run_id: str - bundle: str | None = None - filters: EvaluationFilters = Field(default_factory=EvaluationFilters) - limit: int = 5000 - - -class EvaluationResult(BaseModel): - target_count: int - computed_target_count: int - items_url: str - - -class EvaluationResponse(BaseModel): - status: str - cache_status: str - elapsed_ms: float - result: EvaluationResult - - -class ComparisonSide(BaseModel): - dataset_id: str - run_id: str - bundle: str | None = None - filters: EvaluationFilters = Field(default_factory=EvaluationFilters) - - -class CompareRequest(BaseModel): - a: ComparisonSide - b: ComparisonSide - top_n: int = Field(default=25, ge=1, le=200) - - -class CompareSideMetadata(BaseModel): - dataset_id: str - run_id: str - bundle: str - target_count: int - computed_target_count: int - metrics: SummaryMetrics - provenance: ProvenanceInfo - - -class CompareTargetRow(BaseModel): - target_id: int | None = None - target_name: str - variable: str - geo_level: str | None = None - geographic_id: str | None = None - target_value_a: float | None = None - target_value_b: float | None = None - pe_aggregate_a: float | None = None - pe_aggregate_b: float | None = None - rel_error_a: float | None = None - rel_error_b: float | None = None - abs_rel_error_a: float | None = None - abs_rel_error_b: float | None = None - delta_abs_rel_error: float | None = None - included_in_loss_a: bool - included_in_loss_b: bool - computed_from_bundle_a: str | None = None - computed_from_bundle_b: str | None = None - - -class CompareVariableRow(BaseModel): - variable: str - target_count: int - mean_abs_rel_error_a: float | None = None - mean_abs_rel_error_b: float | None = None - mean_delta_abs_rel_error: float | None = None - improved_count: int - regressed_count: int - - -class CompareResponse(BaseModel): - a: CompareSideMetadata - b: CompareSideMetadata - matched_target_count: int - computed_pair_count: int - improved_count: int - regressed_count: int - improved: list[CompareTargetRow] - regressed: list[CompareTargetRow] - by_variable: list[CompareVariableRow] diff --git a/backend/api/v1/router.py b/backend/api/v1/router.py deleted file mode 100644 index b931275..0000000 --- a/backend/api/v1/router.py +++ /dev/null @@ -1,726 +0,0 @@ -"""Stable v1 API routes for programmatic calibration diagnostics.""" - -from __future__ import annotations - -from pathlib import Path -from time import perf_counter -from typing import Annotated - -import numpy as np -import pandas as pd -from fastapi import APIRouter, HTTPException, Query, Request - -from backend.api.v1.models import ( - BundleItem, - BundleListResponse, - CompareRequest, - CompareResponse, - CompareSideMetadata, - CompareTargetRow, - CompareVariableRow, - ComparisonSide, - DatasetItem, - DatasetListResponse, - EvaluationRequest, - EvaluationResponse, - EvaluationResult, - ProvenanceInfo, - RunItem, - RunListResponse, - SummaryMetrics, - SummaryResponse, - TargetItem, - TargetListResponse, -) -from backend.services import runs as runs_service -from backend.services.bundle_availability import published_bundles -from backend.services.geo_utils import ( - STATE_FIPS_TO_ABBREV, - runtime_dataset_bundle_for, - state_name, -) -from backend.state import AppState - -router = APIRouter(prefix="/api/v1", tags=["api-v1"]) - - -@router.get("/datasets") -def list_datasets_v1() -> DatasetListResponse: - return DatasetListResponse( - items=[ - DatasetItem( - dataset_id=d.id, - label=d.label, - repo_id=d.repo_id, - repo_type=d.repo_type, - layout=d.layout, - primary_h5=d.primary_h5, - ) - for d in runs_service.list_datasets() - ] - ) - - -@router.get("/datasets/{dataset_id}/runs") -def list_runs_v1(dataset_id: str) -> RunListResponse: - try: - runs = runs_service.list_runs(dataset_id) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) - return RunListResponse( - dataset_id=dataset_id, - items=[ - RunItem( - dataset_id=r.dataset_id, - run_id=r.run_id, - label=r.label, - last_modified=r.last_modified, - ) - for r in runs - ], - ) - - -@router.get("/datasets/{dataset_id}/runs/{run_id}/bundles") -def list_bundles_v1( - request: Request, - dataset_id: str, - run_id: str, - kind: str | None = None, - state_fips: int | None = None, - include_target_counts: bool = True, - include_cache_status: bool = True, -) -> BundleListResponse: - dataset = _get_dataset_or_404(dataset_id) - bundles = sorted(published_bundles(dataset.repo_id, run_id)) - state = ( - _load_state(request, dataset_id, run_id) - if include_target_counts or include_cache_status - else None - ) - counts = _bundle_target_counts(state, bundles) if state is not None else {} - included_counts = ( - _bundle_target_counts(state, bundles, included_only=True) - if state is not None - else {} - ) - - items: list[BundleItem] = [] - for bundle in bundles: - item = _bundle_item(bundle) - if kind and item.kind != kind: - continue - if state_fips is not None: - if item.kind != "state" or item.geography_id != str(state_fips): - continue - if include_target_counts: - item.target_count = counts.get(bundle, 0) - item.included_target_count = included_counts.get(bundle, 0) - if include_cache_status: - item.cache_status = _bundle_cache_status(dataset.repo_id, run_id, bundle) - items.append(item) - - return BundleListResponse(dataset_id=dataset_id, run_id=run_id, items=items) - - -@router.get("/datasets/{dataset_id}/runs/{run_id}/summary") -def get_summary_v1( - request: Request, - dataset_id: str, - run_id: str, - bundle: str | None = None, - included: bool | None = None, - geo_level: Annotated[list[str] | None, Query()] = None, - state_fips: Annotated[list[int] | None, Query()] = None, -) -> SummaryResponse: - dataset = _get_dataset_or_404(dataset_id) - state = _load_state(request, dataset_id, run_id) - active_bundle = bundle or dataset.primary_h5 - df = _prepare_targets( - state, - dataset, - run_id, - bundle=active_bundle if bundle else None, - geo_level=geo_level, - state_fips=state_fips, - included=included, - ) - return _summary_response( - state, - dataset, - run_id, - active_bundle, - df, - ) - - -@router.get("/datasets/{dataset_id}/runs/{run_id}/targets") -def list_targets_v1( - request: Request, - dataset_id: str, - run_id: str, - bundle: str | None = None, - geo_level: Annotated[list[str] | None, Query()] = None, - state_fips: Annotated[list[int] | None, Query()] = None, - geographic_id: str | None = None, - variable: Annotated[list[str] | None, Query()] = None, - source: Annotated[list[str] | None, Query()] = None, - included: bool | None = None, - min_abs_rel_error: float | None = None, - sort: str = "abs_rel_error", - order: str = "desc", - limit: Annotated[int, Query(ge=1, le=5000)] = 50, - offset: Annotated[int, Query(ge=0)] = 0, -) -> TargetListResponse: - dataset = _get_dataset_or_404(dataset_id) - state = _load_state(request, dataset_id, run_id) - active_bundle = bundle or dataset.primary_h5 - df = _prepare_targets( - state, - dataset, - run_id, - bundle=active_bundle if bundle else None, - geo_level=geo_level, - state_fips=state_fips, - geographic_id=geographic_id, - variable=variable, - source=source, - included=included, - min_abs_rel_error=min_abs_rel_error, - ) - total = len(df) - if sort in df.columns: - df = df.sort_values(sort, ascending=(order == "asc")) - page = df.iloc[offset : offset + limit] - return TargetListResponse( - dataset_id=dataset_id, - run_id=run_id, - bundle=active_bundle, - items=[_target_item(row, active_bundle) for _, row in page.iterrows()], - total=total, - offset=offset, - limit=limit, - ) - - -@router.get("/datasets/{dataset_id}/runs/{run_id}/targets/{target_id}") -def get_target_v1( - request: Request, - dataset_id: str, - run_id: str, - target_id: int, - bundle: str | None = None, -) -> TargetItem: - dataset = _get_dataset_or_404(dataset_id) - state = _load_state(request, dataset_id, run_id) - active_bundle = bundle or dataset.primary_h5 - df = _prepare_targets( - state, - dataset, - run_id, - bundle=active_bundle if bundle else None, - ) - match = df[df["target_id"].astype("Int64") == target_id] - if match.empty: - raise HTTPException(status_code=404, detail=f"Unknown target_id: {target_id}") - return _target_item(match.iloc[0], active_bundle) - - -@router.post("/evaluate") -def evaluate_v1( - request: Request, - payload: EvaluationRequest, -) -> EvaluationResponse: - start = perf_counter() - dataset = _get_dataset_or_404(payload.dataset_id) - state = _load_state(request, payload.dataset_id, payload.run_id) - active_bundle = payload.bundle or dataset.primary_h5 - filters = payload.filters - before_status = _bundle_cache_status(dataset.repo_id, payload.run_id, active_bundle) - df = _prepare_targets( - state, - dataset, - payload.run_id, - bundle=active_bundle if payload.bundle else None, - geo_level=filters.geo_level, - state_fips=filters.state_fips, - geographic_id=filters.geographic_id, - variable=filters.variable, - source=filters.source, - included=filters.included, - min_abs_rel_error=filters.min_abs_rel_error, - ) - limited = df.head(max(0, payload.limit)) - computed = int( - np.isfinite(limited.get("estimate", pd.Series(dtype=float)).to_numpy(dtype=float)).sum() - ) - after_status = _bundle_cache_status(dataset.repo_id, payload.run_id, active_bundle) - params = [f"bundle={active_bundle}"] - if filters.included is not None: - params.append(f"included={str(filters.included).lower()}") - for level in filters.geo_level or []: - params.append(f"geo_level={level}") - for fips in filters.state_fips or []: - params.append(f"state_fips={fips}") - query = "&".join(params) - items_url = ( - f"/api/v1/datasets/{payload.dataset_id}/runs/{payload.run_id}/targets" - f"?{query}" - ) - return EvaluationResponse( - status="complete", - cache_status=after_status if after_status != "not_computed" else before_status, - elapsed_ms=(perf_counter() - start) * 1000, - result=EvaluationResult( - target_count=int(len(limited)), - computed_target_count=computed, - items_url=items_url, - ), - ) - - -@router.post("/compare") -def compare_v1( - request: Request, - payload: CompareRequest, -) -> CompareResponse: - side_a = _comparison_side(request, payload.a) - side_b = _comparison_side(request, payload.b) - merged = _merge_comparison_targets(side_a["df"], side_b["df"]) - if merged.empty: - raise HTTPException( - status_code=400, - detail="No compatible targets matched between comparison sides.", - ) - - valid = merged.dropna(subset=["abs_rel_error_a", "abs_rel_error_b"]).copy() - if valid.empty: - computed_pair_count = 0 - improved_rows: list[CompareTargetRow] = [] - regressed_rows: list[CompareTargetRow] = [] - by_variable: list[CompareVariableRow] = [] - improved_count = 0 - regressed_count = 0 - else: - valid["delta_abs_rel_error"] = ( - valid["abs_rel_error_b"] - valid["abs_rel_error_a"] - ) - computed_pair_count = int(len(valid)) - improved = valid[valid["delta_abs_rel_error"] < 0] - regressed = valid[valid["delta_abs_rel_error"] > 0] - improved_count = int(len(improved)) - regressed_count = int(len(regressed)) - improved_rows = [ - _compare_target_row(row, side_a["bundle"], side_b["bundle"]) - for _, row in improved.nsmallest( - payload.top_n, - "delta_abs_rel_error", - ).iterrows() - ] - regressed_rows = [ - _compare_target_row(row, side_a["bundle"], side_b["bundle"]) - for _, row in regressed.nlargest( - payload.top_n, - "delta_abs_rel_error", - ).iterrows() - ] - by_variable = _compare_by_variable(valid, payload.top_n) - - return CompareResponse( - a=_compare_side_metadata(side_a), - b=_compare_side_metadata(side_b), - matched_target_count=int(len(merged)), - computed_pair_count=computed_pair_count, - improved_count=improved_count, - regressed_count=regressed_count, - improved=improved_rows, - regressed=regressed_rows, - by_variable=by_variable, - ) - - -def _get_dataset_or_404(dataset_id: str): - try: - return runs_service.get_dataset(dataset_id) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) - - -def _load_state(request: Request, dataset_id: str, run_id: str) -> AppState: - try: - return request.app.state.registry.get(dataset_id, run_id) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) - except Exception as exc: - raise HTTPException( - status_code=500, - detail=f"Failed to load run {dataset_id}/{run_id}: {exc}", - ) - - -def _available_bundles(dataset, run_id: str) -> frozenset[str]: - if dataset.layout not in {"staging", "root", "staging-root"}: - return frozenset() - return published_bundles(dataset.repo_id, run_id) - - -def _prepare_targets( - state: AppState, - dataset, - run_id: str, - *, - bundle: str | None = None, - geo_level: list[str] | None = None, - state_fips: list[int] | None = None, - geographic_id: str | None = None, - variable: list[str] | None = None, - source: list[str] | None = None, - included: bool | None = None, - min_abs_rel_error: float | None = None, -) -> pd.DataFrame: - from backend.routes.targets import _apply_target_filters - from backend.services.bundle_eval import evaluate_bundle - - available = _available_bundles(dataset, run_id) - df = _apply_target_filters( - state.targets_enriched, - variables=variable, - geo_levels=geo_level, - sources=source, - geographic_id=geographic_id, - state_fips=state_fips, - included_only=included, - min_abs_rel_error=min_abs_rel_error, - dataset_files=[bundle] if bundle else None, - available_bundles=available, - ).copy() - - if ( - bundle - and bundle in available - and bundle != dataset.primary_h5 - and not df.empty - ): - df = evaluate_bundle( - df, - repo_id=dataset.repo_id, - run_id=run_id, - bundle=bundle, - time_period=state.time_period, - ) - return df - - -def _comparison_side(request: Request, side: ComparisonSide) -> dict: - dataset = _get_dataset_or_404(side.dataset_id) - state = _load_state(request, side.dataset_id, side.run_id) - active_bundle = side.bundle or dataset.primary_h5 - filters = side.filters - df = _prepare_targets( - state, - dataset, - side.run_id, - bundle=active_bundle if side.bundle else None, - geo_level=filters.geo_level, - state_fips=filters.state_fips, - geographic_id=filters.geographic_id, - variable=filters.variable, - source=filters.source, - included=filters.included, - min_abs_rel_error=filters.min_abs_rel_error, - ) - return { - "dataset": dataset, - "state": state, - "run_id": side.run_id, - "bundle": active_bundle, - "df": df, - } - - -def _merge_comparison_targets(df_a: pd.DataFrame, df_b: pd.DataFrame) -> pd.DataFrame: - keep = [ - "target_id", - "target_name", - "variable", - "geo_level", - "geographic_id", - "value", - "estimate", - "rel_error", - "abs_rel_error", - "included", - ] - a = df_a[[col for col in keep if col in df_a.columns]].dropna( - subset=["target_id"], - ) - b = df_b[[col for col in keep if col in df_b.columns]].dropna( - subset=["target_id"], - ) - a = a.rename(columns={col: f"{col}_a" for col in keep if col != "target_id"}) - b = b.rename(columns={col: f"{col}_b" for col in keep if col != "target_id"}) - merged = a.merge(b, on="target_id", how="inner") - if merged.empty: - return merged - - compatible = pd.Series(True, index=merged.index) - for field in ("target_name", "variable", "geo_level", "geographic_id"): - left = f"{field}_a" - right = f"{field}_b" - if left in merged.columns and right in merged.columns: - compatible &= ( - merged[left].astype("string").fillna("") - == merged[right].astype("string").fillna("") - ) - return merged[compatible].copy() - - -def _compare_side_metadata(side: dict) -> CompareSideMetadata: - summary = _summary_response( - side["state"], - side["dataset"], - side["run_id"], - side["bundle"], - side["df"], - ) - return CompareSideMetadata( - dataset_id=summary.dataset_id, - run_id=summary.run_id, - bundle=summary.bundle, - target_count=summary.target_universe_count, - computed_target_count=summary.computed_target_count, - metrics=summary.metrics, - provenance=summary.provenance, - ) - - -def _compare_target_row( - row: pd.Series, - bundle_a: str, - bundle_b: str, -) -> CompareTargetRow: - estimate_a = _safe_float(row.get("estimate_a")) - estimate_b = _safe_float(row.get("estimate_b")) - return CompareTargetRow( - target_id=_safe_int(row.get("target_id")), - target_name=str(row.get("target_name_a", "")), - variable=str(row.get("variable_a", "")), - geo_level=_none_if_nan(row.get("geo_level_a")), - geographic_id=( - None if _none_if_nan(row.get("geographic_id_a")) is None - else str(_none_if_nan(row.get("geographic_id_a"))) - ), - target_value_a=_safe_float(row.get("value_a")), - target_value_b=_safe_float(row.get("value_b")), - pe_aggregate_a=estimate_a, - pe_aggregate_b=estimate_b, - rel_error_a=_safe_float(row.get("rel_error_a")), - rel_error_b=_safe_float(row.get("rel_error_b")), - abs_rel_error_a=_safe_float(row.get("abs_rel_error_a")), - abs_rel_error_b=_safe_float(row.get("abs_rel_error_b")), - delta_abs_rel_error=_safe_float(row.get("delta_abs_rel_error")), - included_in_loss_a=bool(row.get("included_a", False)), - included_in_loss_b=bool(row.get("included_b", False)), - computed_from_bundle_a=bundle_a if estimate_a is not None else None, - computed_from_bundle_b=bundle_b if estimate_b is not None else None, - ) - - -def _compare_by_variable( - valid: pd.DataFrame, - top_n: int, -) -> list[CompareVariableRow]: - if valid.empty: - return [] - rows: list[CompareVariableRow] = [] - grouped = valid.groupby("variable_a", dropna=False) - for variable, df in grouped: - deltas = df["delta_abs_rel_error"] - rows.append( - CompareVariableRow( - variable=str(variable), - target_count=int(len(df)), - mean_abs_rel_error_a=_safe_float(df["abs_rel_error_a"].mean()), - mean_abs_rel_error_b=_safe_float(df["abs_rel_error_b"].mean()), - mean_delta_abs_rel_error=_safe_float(deltas.mean()), - improved_count=int((deltas < 0).sum()), - regressed_count=int((deltas > 0).sum()), - ) - ) - return sorted( - rows, - key=lambda row: abs(row.mean_delta_abs_rel_error or 0), - reverse=True, - )[:top_n] - - -def _bundle_target_counts( - state: AppState | None, - available: list[str], - *, - included_only: bool = False, -) -> dict[str, int]: - if state is None or state.targets_enriched.empty: - return {} - df = state.targets_enriched - if included_only: - df = df[df["included"].astype(bool)] - available_set = frozenset(available) - counts: dict[str, int] = {} - for _, row in df.iterrows(): - bundle = runtime_dataset_bundle_for( - row.get("geo_level"), - row.get("geographic_id"), - available=available_set, - ) - counts[bundle] = counts.get(bundle, 0) + 1 - return counts - - -def _bundle_item(bundle: str) -> BundleItem: - if bundle.startswith("states/") and bundle.endswith(".h5"): - code = bundle.removeprefix("states/").removesuffix(".h5") - fips = next( - (f for f, abbrev in STATE_FIPS_TO_ABBREV.items() if abbrev == code), - None, - ) - return BundleItem( - bundle=bundle, - kind="state", - geography_id=str(fips) if fips is not None else None, - geography_name=state_name(fips) if fips is not None else code, - ) - if bundle.startswith("districts/"): - return BundleItem( - bundle=bundle, - kind="district", - geography_name=bundle.removeprefix("districts/").removesuffix(".h5"), - ) - if bundle.startswith("national/"): - return BundleItem(bundle=bundle, kind="national", geography_id="US", geography_name="National") - if bundle.startswith("cities/"): - return BundleItem( - bundle=bundle, - kind="city", - geography_name=bundle.removeprefix("cities/").removesuffix(".h5"), - ) - return BundleItem(bundle=bundle, kind="primary") - - -def _bundle_cache_status( - repo_id: str, - run_id: str, - bundle: str, - cache_root: str = ".artifacts", -) -> str: - repo_slug = repo_id.replace("/", "__") - layout_dir = "root" if run_id == "main" else "staging" - safe = bundle.replace("/", "__") - cache = ( - Path(cache_root) / repo_slug / layout_dir / run_id - / f"{safe}.bundle_estimates.pkl" - ) - return "computed" if cache.exists() else "not_computed" - - -def _summary_response( - state: AppState, - dataset, - run_id: str, - bundle: str, - df: pd.DataFrame, -) -> SummaryResponse: - rel = df.get("rel_error", pd.Series(dtype=float)).to_numpy(dtype=float) - abs_rel = df.get("abs_rel_error", pd.Series(dtype=float)).to_numpy(dtype=float) - finite = np.isfinite(rel) & np.isfinite(abs_rel) - abs_finite = abs_rel[finite] - included = df["included"].astype(bool) if "included" in df.columns else pd.Series(False, index=df.index) - loss_available = bool(included.any()) - included_rel = rel[included.to_numpy(dtype=bool)] if len(rel) else np.array([]) - included_rel = included_rel[np.isfinite(included_rel)] - return SummaryResponse( - dataset_id=state.dataset_id, - run_id=run_id, - bundle=bundle, - target_universe_count=int(len(df)), - included_target_count=int(included.sum()), - computed_target_count=int(finite.sum()), - loss_contribution_available=loss_available, - metrics=SummaryMetrics( - median_abs_rel_error=_safe_float(np.median(abs_finite)) if len(abs_finite) else None, - mean_abs_rel_error=_safe_float(np.mean(abs_finite)) if len(abs_finite) else None, - p95_abs_rel_error=_safe_float(np.percentile(abs_finite, 95)) if len(abs_finite) else None, - total_loss=( - _safe_float(np.sum(included_rel ** 2)) - if loss_available and len(included_rel) - else None - ), - ), - provenance=_provenance(dataset, bundle), - ) - - -def _target_item(row: pd.Series, bundle: str) -> TargetItem: - included = bool(row.get("included", False)) - estimate = _safe_float(row.get("estimate")) - return TargetItem( - target_id=_safe_int(row.get("target_id")), - target_name=str(row.get("target_name", "")), - variable=str(row.get("variable", "")), - geo_level=_none_if_nan(row.get("geo_level")), - geographic_id=( - None if _none_if_nan(row.get("geographic_id")) is None - else str(_none_if_nan(row.get("geographic_id"))) - ), - target_value=_safe_float(row.get("value")), - pe_aggregate=estimate, - rel_error=_safe_float(row.get("rel_error")), - abs_rel_error=_safe_float(row.get("abs_rel_error")), - included_in_loss=included, - loss_contribution=_safe_float(row.get("loss_contribution")) if included else None, - computed_from_bundle=bundle if estimate is not None else None, - target_value_source="policy_data.db", - included_source="unified_diagnostics.csv" if included else None, - calibration_pattern_source=None, - eval_note=_none_if_nan(row.get("eval_note")), - ) - - -def _provenance(dataset, bundle: str) -> ProvenanceInfo: - if dataset.layout == "root": - diagnostics = "calibration/logs/unified_diagnostics.csv" - elif dataset.layout == "staging-root": - diagnostics = None - else: - diagnostics = "calibration/runs//diagnostics/unified_diagnostics.csv" - return ProvenanceInfo( - target_db="policy_data.db", - diagnostics=diagnostics, - aggregate_source=bundle, - calibration_pattern_source=None, - ) - - -def _safe_float(value) -> float | None: - if value is None or pd.isna(value): - return None - try: - out = float(value) - except (TypeError, ValueError): - return None - return out if np.isfinite(out) else None - - -def _safe_int(value) -> int | None: - if value is None or pd.isna(value): - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _none_if_nan(value): - if value is None or pd.isna(value): - return None - return value diff --git a/backend/app.py b/backend/app.py deleted file mode 100644 index eb5ca31..0000000 --- a/backend/app.py +++ /dev/null @@ -1,110 +0,0 @@ -"""FastAPI application factory. - -The backend hosts a registry of loaded calibration runs; each request -selects a run via ?dataset & ?run query params. See backend.state.get_state. -""" - -import logging -import os -from contextlib import asynccontextmanager - -from fastapi import FastAPI, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse - -from backend.api.v1.router import router as api_v1_router -from backend.routes import ( - analysis, - compare, - geography, - microplex, - nodes, - pipeline, - runs as runs_route, - strata, - summary, - target_inventory, - targets, - weights, -) -from backend.services import runs as runs_service -from backend.services.registry import RunRegistry - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(levelname)s - %(message)s", -) -logger = logging.getLogger(__name__) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - cache_size = int(os.environ.get("RUN_CACHE_SIZE", "3")) - app.state.registry = RunRegistry(max_size=cache_size) - - # Optional warm-load of a default run, so /health and existing endpoints - # work without query params during development. - default = runs_service.default_selection() - if default is not None: - dataset_id, run_id = default - logger.info("Warming default run %s/%s", dataset_id, run_id) - try: - app.state.registry.get(dataset_id, run_id) - except Exception: - logger.exception("Failed to warm-load default run") - else: - logger.info( - "No DEFAULT_DATASET/DEFAULT_RUN set; clients must pass " - "?dataset & ?run query params." - ) - yield - - -app = FastAPI( - title="Calibration Diagnostics API", - description="Interactive diagnostics for survey weight calibration", - lifespan=lifespan, -) - -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_methods=["*"], - allow_headers=["*"], -) - -app.include_router(runs_route.router, tags=["runs"]) -app.include_router(api_v1_router) -app.include_router(summary.router, tags=["summary"]) -app.include_router(pipeline.router, tags=["pipeline"]) -app.include_router(target_inventory.router, tags=["target-inventory"]) -app.include_router(nodes.router, tags=["nodes"]) -app.include_router(analysis.router, prefix="/analysis", tags=["analysis"]) -app.include_router(compare.router, tags=["compare"]) -app.include_router(microplex.router, tags=["microplex"]) -app.include_router(geography.router, prefix="/geography", tags=["geography"]) -app.include_router(targets.router, prefix="/targets", tags=["targets"]) -app.include_router(strata.router, prefix="/strata", tags=["strata"]) -app.include_router(weights.router, prefix="/weights", tags=["weights"]) - - -@app.get("/health") -def health(request: Request): - registry: RunRegistry = request.app.state.registry - return { - "status": "ok", - "loaded_runs": [ - {"dataset": d, "run": r} for d, r in registry.loaded_keys() - ], - "default": runs_service.default_selection(), - } - - -@app.exception_handler(ValueError) -async def value_error_handler(request: Request, exc: ValueError): - return JSONResponse(status_code=400, content={"detail": str(exc)}) - - -@app.exception_handler(IndexError) -async def index_error_handler(request: Request, exc: IndexError): - return JSONResponse(status_code=404, content={"detail": "Index out of range"}) diff --git a/backend/data/pipeline/microplex.json b/backend/data/pipeline/microplex.json deleted file mode 100644 index ddd7f0d..0000000 --- a/backend/data/pipeline/microplex.json +++ /dev/null @@ -1,451 +0,0 @@ -{ - "pipeline_id": "microplex-us", - "pipeline_label": "Microplex-US", - "description": "US-specific Microplex runtime: source fusion, donor integration, synthesis, PolicyEngine-compatible export, calibration, and oracle evaluation.", - "source_repo": "PolicyEngine/microplex-us", - "source_urls": [ - "https://github.com/PolicyEngine/microplex-us", - "https://github.com/PolicyEngine/microplex-us/blob/main/docs/architecture.md", - "https://github.com/PolicyEngine/microplex-us/blob/main/docs/pipeline-stages.md", - "https://github.com/PolicyEngine/microplex-us/blob/main/docs/stage-contracts.md", - "https://github.com/PolicyEngine/microplex-us/blob/main/docs/benchmarking.md", - "https://github.com/PolicyEngine/microplex-us/blob/main/docs/policyengine-oracle-compatibility.md" - ], - "nodes": [ - { - "id": "run_profile", - "label": "Resolve run profile", - "description": "Resolve the Microplex runtime profile, target period, source bundle, calibration backend, random seed, and artifact root.", - "node_type": "entrypoint", - "status": "current", - "stage_id": "01_run_profile", - "pathways": ["microplex_us"], - "source_file": "microplex_us/pipelines/us.py", - "artifacts_in": ["runtime_overrides", "profile_defaults"], - "artifacts_out": ["resolved_config", "provider_query_plan", "manifest"], - "implementation_refs": [ - "USMicroplexPipeline", - "default_policyengine_us_data_rebuild_config", - "run manifest" - ], - "explanation": "This fixes the build contract before data is loaded: sources, period, target profile, calibration mode, artifact location, and reproducibility settings.", - "analyst_questions": [ - "Which profile is this run using?", - "Which sources and target period are in scope?", - "Is this an incumbent-compatibility run or a challenger mode?" - ] - }, - { - "id": "source_providers", - "label": "Load US source providers", - "description": "Load CPS, PUF, and donor-source adapters into canonical source descriptors and observation frames.", - "node_type": "source", - "status": "current", - "stage_id": "02_source_loading", - "pathways": ["microplex_us"], - "source_file": "microplex_us/pipelines/us.py", - "artifacts_in": ["resolved_config", "provider_query_plan", "raw_cps", "raw_puf", "donor_sources", "source_metadata"], - "artifacts_out": ["observation_frames", "source_descriptors"], - "implementation_refs": [ - "microplex_us.pipelines.USMicroplexPipeline", - "microplex_us.data_sources.*" - ], - "explanation": "This is the boundary where survey-specific files become Microplex-owned tables with explicit variable, entity, and source capability metadata.", - "analyst_questions": [ - "Which source owns each variable?", - "Which variables are native to the scaffold vs donor-only?", - "Are source units and entity IDs preserved?" - ] - }, - { - "id": "fusion_plan", - "label": "Build fusion plan", - "description": "Construct a plan for combining scaffold and donor sources based on variable capability metadata and declared entity semantics.", - "node_type": "planner", - "status": "current", - "stage_id": "03_source_planning", - "pathways": ["microplex_us"], - "source_file": "microplex_us/pipelines/us.py", - "artifacts_in": ["observation_frames", "source_descriptors"], - "artifacts_out": ["fusion_plan"], - "implementation_refs": [ - "FusionPlan", - "source capability registry", - "variable semantic registry" - ], - "explanation": "The fusion plan decides what can be copied, modeled, projected across entities, or deferred. It is the attribution layer for later data differences.", - "analyst_questions": [ - "Which variables are directly observed in the scaffold?", - "Which variables require donor modeling?", - "What entity projection is needed?" - ] - }, - { - "id": "scaffold_seed", - "label": "Choose scaffold and seed population", - "description": "Select a public structured scaffold source and prepare canonical seed rows for the synthetic population.", - "node_type": "process", - "status": "current", - "stage_id": "04_seed_scaffold", - "pathways": ["microplex_us"], - "source_file": "microplex_us/pipelines/us.py", - "artifacts_in": ["fusion_plan", "observation_frames"], - "artifacts_out": ["seed_population"], - "implementation_refs": [ - "USMicroplexPipeline", - "canonical seed data preparation" - ], - "explanation": "The scaffold gives the synthetic dataset its initial row structure, entity IDs, and observed baseline features.", - "analyst_questions": [ - "Is the scaffold representative for the target geography and period?", - "Which later variables depend on scaffold entity IDs?" - ] - }, - { - "id": "donor_blocks", - "label": "Integrate donor-only variables", - "description": "Fit and generate donor-source variable blocks using declared match strategies, condition surfaces, and entity projection rules.", - "node_type": "model", - "status": "current", - "stage_id": "05_donor_integration_synthesis", - "pathways": ["microplex_us"], - "source_file": "microplex_us/pe_source_impute_engine.py", - "artifacts_in": ["seed_population", "fusion_plan", "donor_sources"], - "artifacts_out": ["imputed_variable_blocks", "imputation_sidecars"], - "implementation_refs": [ - "pe_source_impute_blocks.json", - "pe_source_impute_engine.py", - "donor_surveys.py" - ], - "explanation": "This is where CPS/PUF scaffold rows get variables that only exist in donor sources. It is one of the highest-risk points for downstream tax and benefit aggregates.", - "analyst_questions": [ - "Which donor block supplied a variable?", - "What conditions were used for matching?", - "Did projection from person/tax-unit/household change totals?" - ] - }, - { - "id": "synthesize_population", - "label": "Synthesize population", - "description": "Materialize a complete synthetic population after scaffold and donor integration.", - "node_type": "model", - "status": "current", - "stage_id": "05_donor_integration_synthesis", - "pathways": ["microplex_us"], - "source_file": "microplex_us/pipelines/us.py", - "artifacts_in": ["seed_population", "imputed_variable_blocks"], - "artifacts_out": ["synthetic_population"], - "implementation_refs": [ - "generic microplex synthesis engine", - "USMicroplexPipeline" - ], - "explanation": "This produces the candidate microdata before PolicyEngine-specific shaping, calibration, and evaluation.", - "analyst_questions": [ - "How many synthetic records were produced?", - "Do donor variables preserve valid combinations?", - "Are missingness and support flags auditable?" - ] - }, - { - "id": "pe_entity_tables", - "label": "Build PolicyEngine entity tables", - "description": "Convert the synthetic population into PolicyEngine-style person, tax-unit, spm-unit, household, and marital-unit tables.", - "node_type": "export", - "status": "current", - "stage_id": "06_policyengine_entities", - "pathways": ["policyengine_compatibility"], - "source_file": "microplex_us/policyengine/us.py", - "artifacts_in": ["synthetic_population"], - "artifacts_out": ["pe_entity_tables"], - "implementation_refs": [ - "microplex_us.policyengine.us", - "PE-US-compatible export" - ], - "explanation": "PolicyEngine can only evaluate aggregates correctly if the entity graph and variable placement match its expected H5 schema.", - "analyst_questions": [ - "Are entity IDs and joins valid?", - "Are variables stored on the correct entity?", - "Which values are native vs derived?" - ] - }, - { - "id": "pe_derived_features", - "label": "Materialize PE-derived features", - "description": "Run PolicyEngine-compatible materialization for derived features required by calibration targets and diagnostics.", - "node_type": "policyengine", - "status": "current", - "stage_id": "06_policyengine_entities", - "pathways": ["policyengine_compatibility"], - "source_file": "microplex_us/policyengine/us.py", - "artifacts_in": ["pe_entity_tables", "policyengine_us"], - "artifacts_out": ["pe_feature_tables"], - "implementation_refs": [ - "policyengine-us", - "microplex_us.policyengine.us" - ], - "explanation": "This turns raw modeled inputs into the same measured variables used by the target oracle.", - "analyst_questions": [ - "Which variables are stored inputs vs PE formulas?", - "Did a missing node prevent a target from being evaluated?", - "Are formula-period assumptions aligned?" - ] - }, - { - "id": "target_provider", - "label": "Load active PE target DB", - "description": "Load the active PolicyEngine-US target database and compile canonical target rows for evaluation and calibration planning.", - "node_type": "target", - "status": "current", - "stage_id": "07_calibration", - "pathways": ["policyengine_oracle"], - "source_file": "microplex_us/policyengine/harness.py", - "artifacts_in": ["policy_data_db"], - "artifacts_out": ["active_target_set", "compiled_target_specs"], - "implementation_refs": [ - "PolicyEngineUSDBTargetProvider", - "policyengine-us-data targets DB" - ], - "explanation": "In Microplex benchmarking, the target DB is treated as truth. The incumbent us-data dataset is a comparator, not the oracle.", - "analyst_questions": [ - "Which targets are active?", - "Which targets are supported by the candidate dataset?", - "Which targets are calibration constraints vs audit-only?" - ] - }, - { - "id": "calibration_planner", - "label": "Plan calibration constraints", - "description": "Classify target rows into solve-now, solve-later, and audit-only groups instead of flattening the entire DB into one solve.", - "node_type": "planner", - "status": "current", - "stage_id": "07_calibration", - "pathways": ["policyengine_oracle"], - "source_file": "microplex_us/policyengine/harness.py", - "artifacts_in": ["compiled_target_specs", "pe_feature_tables"], - "artifacts_out": ["calibration_plan"], - "implementation_refs": [ - "full-oracle loss with unsupported-target penalty", - "solve_now / solve_later / audit_only classification" - ], - "explanation": "This keeps every target visible while avoiding the claim that every target row is simultaneously solvable in one numerical step.", - "analyst_questions": [ - "Which relevant targets are unsupported?", - "Which targets are optimized now?", - "Which targets only inform diagnostics?" - ] - }, - { - "id": "calibrate_weights", - "label": "Calibrate / reweight", - "description": "Adjust synthetic population weights against the planned PE-US targets, including narrow deferred passes when they improve full-oracle score.", - "node_type": "optimizer", - "status": "current", - "stage_id": "07_calibration", - "pathways": ["policyengine_oracle"], - "source_file": "microplex_us/pipelines/us.py", - "artifacts_in": ["pe_feature_tables", "calibration_plan"], - "artifacts_out": ["calibrated_population", "calibration_sidecars"], - "implementation_refs": [ - "Microplex calibration planner", - "full active PE-US target estate" - ], - "explanation": "This is the main calibration step: it changes weights or fitted outputs so candidate aggregates get closer to target DB values.", - "analyst_questions": [ - "Which targets drove the largest weight changes?", - "Did calibration improve broad loss or only a narrow family?", - "Are unsupported targets penalized in the final score?" - ] - }, - { - "id": "export_h5", - "label": "Export PE-ingestable H5", - "description": "Write the calibrated candidate dataset in the H5 shape that PolicyEngine-US can load.", - "node_type": "export", - "status": "current", - "stage_id": "08_dataset_assembly", - "pathways": ["policyengine_compatibility"], - "source_file": "microplex_us/pipelines/artifacts.py", - "artifacts_in": ["calibrated_population", "pe_entity_tables"], - "artifacts_out": ["microplex_h5"], - "implementation_refs": [ - "artifact bundle", - "PE-ingestable H5 export" - ], - "explanation": "The exported H5 is the thing that can be compared with us-data in a shared PolicyEngine measurement harness.", - "analyst_questions": [ - "Can PolicyEngine load the H5 without schema repairs?", - "Are all expected entity arrays present?", - "Is the exported artifact published or private?" - ] - }, - { - "id": "baseline_us_data", - "label": "Load incumbent us-data baseline", - "description": "Load the comparable policyengine-us-data dataset slice used as the incumbent baseline.", - "node_type": "baseline", - "status": "current", - "stage_id": "09_validation_benchmarking", - "pathways": ["policyengine_oracle"], - "source_file": "microplex_us/policyengine/comparison.py", - "artifacts_in": ["policyengine_us_data_h5"], - "artifacts_out": ["baseline_dataset"], - "implementation_refs": [ - "policyengine-us-data incumbent comparator", - "baselineSlice" - ], - "explanation": "This is not truth; it is the incumbent comparator run through the same target oracle as Microplex.", - "analyst_questions": [ - "Which us-data artifact is the baseline?", - "Does it use the same target period and profile?", - "Are both datasets evaluated through the same PE version?" - ] - }, - { - "id": "oracle_evaluation", - "label": "Evaluate through PE target oracle", - "description": "Run Microplex and us-data through PolicyEngine-US and compare implied aggregates to the same active target DB.", - "node_type": "evaluation", - "status": "current", - "stage_id": "09_validation_benchmarking", - "pathways": ["policyengine_oracle"], - "source_file": "microplex_us/policyengine/comparison.py", - "artifacts_in": ["microplex_h5", "baseline_dataset", "active_target_set"], - "artifacts_out": ["policyengine_harness", "target_diagnostics"], - "implementation_refs": [ - "microplex_us.policyengine.harness", - "microplex_us.policyengine.comparison" - ], - "explanation": "This produces composite parity loss, mean absolute relative error, supported target rate, target win rate, and per-target deltas when diagnostics are available.", - "analyst_questions": [ - "Which target families improved?", - "Which target families regressed?", - "Is the gain broad or concentrated in a few target weights?" - ] - }, - { - "id": "artifact_bundle", - "label": "Save artifacts and run index", - "description": "Persist the versioned artifact bundle, harness JSON, registry rows, DuckDB index, and diagnostic sidecars.", - "node_type": "artifact", - "status": "current", - "stage_id": "09_validation_benchmarking", - "pathways": ["frontier_tracking"], - "source_file": "microplex_us/pipelines/artifacts.py", - "artifacts_in": ["policyengine_harness", "target_diagnostics", "microplex_h5"], - "artifacts_out": ["artifact_bundle", "run_registry_jsonl", "run_index_duckdb"], - "implementation_refs": [ - "policyengine_harness.json", - "policyengine_native_scores.json", - "pe_us_data_rebuild_native_audit.json", - "run_registry.jsonl", - "run_index.duckdb", - "manifest.artifacts.policyengine_native_target_diagnostics", - "pe_native_target_diagnostics.json" - ], - "explanation": "This is the reproducibility and frontier-analysis layer. Newer Microplex runs record full target diagnostics as manifest.artifacts.policyengine_native_target_diagnostics pointing to pe_native_target_diagnostics.json. The current calibration-diagnostics app only reads public committed JSON summaries unless a generated artifact root or artifact service is supplied.", - "analyst_questions": [ - "Which artifacts are public?", - "Which artifacts are generated but not committed?", - "Can this app ingest full per-target diagnostics yet?" - ] - }, - { - "id": "dashboard_public_json", - "label": "Publish dashboard JSON summaries", - "description": "Commit aggregate parity, regression, and IRS drilldown JSONs that this dashboard can read without credentials.", - "node_type": "publication", - "status": "current", - "stage_id": "09_validation_benchmarking", - "pathways": ["frontier_tracking"], - "source_file": "artifacts/", - "artifacts_in": ["artifact_bundle", "policyengine_harness", "target_diagnostics"], - "artifacts_out": ["parity_json", "regression_summary_json", "irs_drilldown_json"], - "implementation_refs": [ - "pe_us_data_rebuild_parity.json", - "live_pe_us_data_rebuild_checkpoint_modelpass_regression_summary_20260410.json", - "live_pe_us_data_rebuild_checkpoint_national_irs_other_drilldown_20260410.json" - ], - "explanation": "This is the narrow public surface currently shown on the Microplex dashboard page.", - "analyst_questions": [ - "Do these summaries contain enough detail for a target-level diagnosis?", - "Which run produced the public headline?", - "When should the dashboard switch to full diagnostics?" - ] - } - ], - "edges": [ - {"from": "run_profile", "to": "source_providers", "artifact": "provider_query_plan", "kind": "config"}, - {"from": "run_profile", "to": "target_provider", "artifact": "resolved_config", "kind": "config"}, - {"from": "source_providers", "to": "fusion_plan", "artifact": "observation_frames", "kind": "data"}, - {"from": "fusion_plan", "to": "scaffold_seed", "artifact": "fusion_plan", "kind": "plan"}, - {"from": "scaffold_seed", "to": "donor_blocks", "artifact": "seed_population", "kind": "data"}, - {"from": "fusion_plan", "to": "donor_blocks", "artifact": "fusion_plan", "kind": "plan"}, - {"from": "donor_blocks", "to": "synthesize_population", "artifact": "imputed_variable_blocks", "kind": "data"}, - {"from": "scaffold_seed", "to": "synthesize_population", "artifact": "seed_population", "kind": "data"}, - {"from": "synthesize_population", "to": "pe_entity_tables", "artifact": "synthetic_population", "kind": "data"}, - {"from": "pe_entity_tables", "to": "pe_derived_features", "artifact": "pe_entity_tables", "kind": "data"}, - {"from": "target_provider", "to": "calibration_planner", "artifact": "compiled_target_specs", "kind": "target"}, - {"from": "pe_derived_features", "to": "calibration_planner", "artifact": "pe_feature_tables", "kind": "data"}, - {"from": "calibration_planner", "to": "calibrate_weights", "artifact": "calibration_plan", "kind": "plan"}, - {"from": "pe_derived_features", "to": "calibrate_weights", "artifact": "pe_feature_tables", "kind": "data"}, - {"from": "calibrate_weights", "to": "export_h5", "artifact": "calibrated_population", "kind": "data"}, - {"from": "pe_entity_tables", "to": "export_h5", "artifact": "pe_entity_tables", "kind": "data"}, - {"from": "export_h5", "to": "oracle_evaluation", "artifact": "microplex_h5", "kind": "dataset"}, - {"from": "baseline_us_data", "to": "oracle_evaluation", "artifact": "baseline_dataset", "kind": "dataset"}, - {"from": "target_provider", "to": "oracle_evaluation", "artifact": "active_target_set", "kind": "target"}, - {"from": "oracle_evaluation", "to": "artifact_bundle", "artifact": "policyengine_harness", "kind": "diagnostic"}, - {"from": "export_h5", "to": "artifact_bundle", "artifact": "microplex_h5", "kind": "dataset"}, - {"from": "artifact_bundle", "to": "dashboard_public_json", "artifact": "artifact_bundle", "kind": "publication"}, - {"from": "oracle_evaluation", "to": "dashboard_public_json", "artifact": "target_diagnostics", "kind": "diagnostic"} - ], - "unproduced_artifacts": [ - "runtime_overrides", - "profile_defaults", - "raw_cps", - "raw_puf", - "donor_sources", - "source_metadata", - "policyengine_us", - "policy_data_db", - "policyengine_us_data_h5" - ], - "stats": { - "node_count": 16, - "edge_count": 23, - "by_type": { - "entrypoint": 1, - "source": 1, - "planner": 2, - "process": 1, - "model": 2, - "export": 2, - "policyengine": 1, - "target": 1, - "optimizer": 1, - "baseline": 1, - "evaluation": 1, - "artifact": 1, - "publication": 1 - }, - "by_status": { - "current": 16 - }, - "by_pathway": { - "microplex_us": 6, - "policyengine_compatibility": 3, - "policyengine_oracle": 5, - "frontier_tracking": 2 - }, - "by_stage": { - "01_run_profile": 1, - "02_source_loading": 1, - "03_source_planning": 1, - "04_seed_scaffold": 1, - "05_donor_integration_synthesis": 2, - "06_policyengine_entities": 2, - "07_calibration": 3, - "08_dataset_assembly": 1, - "09_validation_benchmarking": 4 - } - } -} diff --git a/backend/data/pipeline/microplex/stages/01_run_profile.md b/backend/data/pipeline/microplex/stages/01_run_profile.md deleted file mode 100644 index 787fa12..0000000 --- a/backend/data/pipeline/microplex/stages/01_run_profile.md +++ /dev/null @@ -1,22 +0,0 @@ -# Run Profile - -This stage fixes the runtime contract before any data is loaded. It resolves the selected Microplex profile, source bundle, target period, calibration backend, random seed, artifact root, and incumbent-comparison mode. - -## Inputs - -- Profile defaults and runtime overrides. -- Target period and target profile. -- Source-provider names and query settings. - -## Outputs - -- Resolved config. -- Provider/query plan. -- Run manifest. - -## Analyst Checks - -- Confirm whether the run is an incumbent-compatibility profile or a challenger mode. -- Confirm that the source bundle and target period match the comparison being shown. -- Confirm that the artifact root and manifest are versioned enough to reproduce the run. - diff --git a/backend/data/pipeline/microplex/stages/02_source_loading.md b/backend/data/pipeline/microplex/stages/02_source_loading.md deleted file mode 100644 index f6d16f3..0000000 --- a/backend/data/pipeline/microplex/stages/02_source_loading.md +++ /dev/null @@ -1,22 +0,0 @@ -# Source Loading - -This stage turns survey-specific files into Microplex observation frames with explicit source descriptors, entity relationships, and variable capability metadata. - -## Inputs - -- Resolved provider/query plan. -- Raw CPS, PUF, and donor-source extracts. -- Source metadata and mappings. - -## Outputs - -- Observation frames. -- Source descriptors. -- Entity relationship metadata. - -## Analyst Checks - -- Identify which source owns each variable. -- Check whether variables are native to the scaffold or donor-only. -- Check that entity IDs and source units are preserved where later projection depends on them. - diff --git a/backend/data/pipeline/microplex/stages/03_source_planning.md b/backend/data/pipeline/microplex/stages/03_source_planning.md deleted file mode 100644 index a240e68..0000000 --- a/backend/data/pipeline/microplex/stages/03_source_planning.md +++ /dev/null @@ -1,22 +0,0 @@ -# Source Planning - -This stage builds the fusion plan. It decides how scaffold and donor sources can be combined based on variable capability metadata, source overlap, entity semantics, and declared projection rules. - -## Inputs - -- Observation frames. -- Source descriptors. -- Variable semantic registry. - -## Outputs - -- Fusion plan. -- Scaffold selection. -- Donor-block plan. - -## Analyst Checks - -- Identify which variables are copied, modeled, projected, or deferred. -- Check donor-block condition surfaces. -- Confirm that group-level variables have explicit projection aggregation rules. - diff --git a/backend/data/pipeline/microplex/stages/04_seed_scaffold.md b/backend/data/pipeline/microplex/stages/04_seed_scaffold.md deleted file mode 100644 index 0611e88..0000000 --- a/backend/data/pipeline/microplex/stages/04_seed_scaffold.md +++ /dev/null @@ -1,21 +0,0 @@ -# Seed Scaffold - -This stage creates the pre-donor seed population from the selected scaffold source. The scaffold provides the initial row structure, entity graph, and observed baseline features. - -## Inputs - -- Source plan. -- Selected scaffold frame. -- Identifier and entity construction rules. - -## Outputs - -- Pre-donor seed frame. -- Scaffold-side metadata. - -## Analyst Checks - -- Check whether the scaffold is appropriate for the target period and geography. -- Check which later variables depend on scaffold entity IDs. -- Check whether row counts and entity relationships are stable before donor integration. - diff --git a/backend/data/pipeline/microplex/stages/05_donor_integration_synthesis.md b/backend/data/pipeline/microplex/stages/05_donor_integration_synthesis.md deleted file mode 100644 index e166f90..0000000 --- a/backend/data/pipeline/microplex/stages/05_donor_integration_synthesis.md +++ /dev/null @@ -1,23 +0,0 @@ -# Donor Integration And Synthesis - -This stage adds donor-only variables and materializes the synthetic candidate population. It is one of the highest-risk stages because donor matching, projection, and support enforcement can move downstream tax and benefit aggregates. - -## Inputs - -- Seed scaffold. -- Donor frames. -- Donor-block manifest and condition surfaces. -- Random seed and support requirements. - -## Outputs - -- Imputed variable blocks. -- Synthetic population. -- Imputation sidecars and source-weight diagnostics. - -## Analyst Checks - -- Identify which donor block supplied a variable. -- Inspect the match conditions used for high-impact variables. -- Check whether entity projection changed totals or invalidated combinations. - diff --git a/backend/data/pipeline/microplex/stages/06_policyengine_entities.md b/backend/data/pipeline/microplex/stages/06_policyengine_entities.md deleted file mode 100644 index d26473d..0000000 --- a/backend/data/pipeline/microplex/stages/06_policyengine_entities.md +++ /dev/null @@ -1,21 +0,0 @@ -# PolicyEngine Entities - -This stage converts the synthetic population into PolicyEngine-compatible entity tables and materializes PE-derived features needed by targets. - -## Inputs - -- Synthetic candidate population. -- PolicyEngine entity mapping rules. -- `policyengine-us` variable definitions. - -## Outputs - -- Person, tax-unit, SPM-unit, household, and marital-unit tables. -- Materialized PolicyEngine input and derived feature tables. - -## Analyst Checks - -- Confirm variables sit on the correct PolicyEngine entity. -- Confirm entity joins are valid. -- Distinguish stored inputs from formula outputs when diagnosing a target. - diff --git a/backend/data/pipeline/microplex/stages/07_calibration.md b/backend/data/pipeline/microplex/stages/07_calibration.md deleted file mode 100644 index 4038586..0000000 --- a/backend/data/pipeline/microplex/stages/07_calibration.md +++ /dev/null @@ -1,23 +0,0 @@ -# Calibration - -This stage loads the active PolicyEngine-US target database, compiles target rows, plans the calibration solve, and reweights or adjusts the candidate population. - -## Inputs - -- PolicyEngine entity/feature tables. -- Active target database. -- Calibration config and full-oracle scoring rules. - -## Outputs - -- Active target set. -- Calibration plan. -- Calibrated population. -- Calibration sidecars. - -## Analyst Checks - -- Separate `solve_now`, `solve_later`, and `audit_only` targets. -- Check unsupported targets and the penalty they contribute. -- Check whether calibration improves broad oracle loss or only a narrow target family. - diff --git a/backend/data/pipeline/microplex/stages/08_dataset_assembly.md b/backend/data/pipeline/microplex/stages/08_dataset_assembly.md deleted file mode 100644 index 6df805d..0000000 --- a/backend/data/pipeline/microplex/stages/08_dataset_assembly.md +++ /dev/null @@ -1,22 +0,0 @@ -# Dataset Assembly - -This stage writes the calibrated candidate into a PolicyEngine-ingestable H5 and records dataset assembly metadata. - -## Inputs - -- Calibrated population. -- PolicyEngine entity tables. -- Export maps and dataset-year metadata. - -## Outputs - -- `policyengine_us.h5` or equivalent Microplex H5. -- Stage manifest. -- Data-flow snapshot and artifact inventory. - -## Analyst Checks - -- Confirm PolicyEngine can load the H5 without schema repair. -- Confirm all required entity arrays are present. -- Check whether the H5 is public, private, or only referenced by summary artifacts. - diff --git a/backend/data/pipeline/microplex/stages/09_validation_benchmarking.md b/backend/data/pipeline/microplex/stages/09_validation_benchmarking.md deleted file mode 100644 index d303080..0000000 --- a/backend/data/pipeline/microplex/stages/09_validation_benchmarking.md +++ /dev/null @@ -1,24 +0,0 @@ -# Validation And Benchmarking - -This stage evaluates Microplex and the incumbent `policyengine-us-data` baseline through the same PolicyEngine target oracle. - -## Inputs - -- Microplex H5. -- Incumbent baseline dataset. -- Active PolicyEngine-US target set. - -## Outputs - -- `policyengine_harness.json`. -- `policyengine_native_scores.json`. -- `pe_us_data_rebuild_native_audit.json`. -- `run_index.duckdb`. -- `pe_native_target_diagnostics.json` in the run bundle, recorded as `manifest.artifacts.policyengine_native_target_diagnostics`, when full row-level diagnostics are generated. -- Public parity, regression, and drilldown JSON summaries. - -## Analyst Checks - -- Treat the target DB as truth; treat `policyengine-us-data` as the incumbent comparator. -- Check target families that improve or regress. -- Remember that this dashboard currently reads public summary JSONs, not the generated H5, run index, or full per-target diagnostic bundle unless a generated artifact root or artifact service is wired in. diff --git a/backend/data/pipeline/nodes.json b/backend/data/pipeline/nodes.json deleted file mode 100644 index f5af133..0000000 --- a/backend/data/pipeline/nodes.json +++ /dev/null @@ -1,2126 +0,0 @@ -{ - "edges": [ - { - "artifact": "worker_bootstrap.json", - "from": "local_h5_worker_bootstrap_bundle", - "kind": "data_flow", - "to": "local_h5_worker_session" - }, - { - "artifact": "bootstrap/{scope}/worker_bootstrap.json", - "from": "local_h5_worker_bootstrap_store", - "kind": "data_flow", - "to": "local_h5_worker_session_factory" - }, - { - "artifact": "bootstrap/{scope}/worker_bootstrap.json", - "from": "local_h5_worker_bootstrap_builder", - "kind": "data_flow", - "to": "local_h5_worker_session_factory" - }, - { - "artifact": "bootstrap/{scope}/entity_graph.npz", - "from": "local_h5_worker_bootstrap_store", - "kind": "data_flow", - "to": "local_h5_worker_session_factory" - }, - { - "artifact": "bootstrap/{scope}/entity_graph.npz", - "from": "local_h5_worker_bootstrap_builder", - "kind": "data_flow", - "to": "local_h5_worker_session_factory" - }, - { - "artifact": "stratified_extended_cps_2024.h5", - "from": "create_stratified", - "kind": "data_flow", - "to": "source_impute" - }, - { - "artifact": "extended_cps_2024.h5", - "from": "record_double", - "kind": "data_flow", - "to": "create_stratified" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "local_h5_worker_bootstrap_bundle" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "local_h5_worker_bootstrap_bundle" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "local_h5_worker_bootstrap_builder" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "local_h5_worker_bootstrap_builder" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "clone_weight_matrix" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "clone_weight_matrix" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "local_h5_worker_session" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "local_h5_worker_session" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "local_h5_worker_session_factory" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "local_h5_worker_session_factory" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "load_calibration_geography" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "load_calibration_geography" - }, - { - "artifact": "calibration_weights.npy", - "from": "fit_model", - "kind": "data_flow", - "to": "build_h5" - }, - { - "artifact": "calibration_weights.npy", - "from": "run_calibration", - "kind": "data_flow", - "to": "build_h5" - }, - { - "artifact": "calibration_package.pkl", - "from": "run_calibration", - "kind": "data_flow", - "to": "init_weights" - }, - { - "artifact": "calibration_package.pkl", - "from": "run_calibration", - "kind": "data_flow", - "to": "fit_model" - }, - { - "artifact": "cps_only_predictions", - "from": "cps_only", - "kind": "data_flow", - "to": "qrf_pass2" - }, - { - "artifact": "extended_cps_stage2", - "from": "qrf_pass2", - "kind": "data_flow", - "to": "formula_drop" - } - ], - "nodes": [ - { - "decorator_line": 37, - "description": "Build typed local H5 requests from US states, districts, and supported city rules.", - "id": "local_h5_area_catalog", - "label": "USAreaCatalog", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/area_catalog.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USAreaCatalog", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_area_catalog.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db" - ], - "artifacts_out": [ - "worker_bootstrap.json", - "entity_graph.npz" - ], - "decorator_line": 52, - "description": "Persisted deterministic worker setup facts for one local H5 scope.", - "id": "local_h5_worker_bootstrap_bundle", - "label": "WorkerBootstrapBundle", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/bootstrap.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "WorkerBootstrapBundle", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_bootstrap.py" - ] - }, - { - "artifacts_out": [ - "bootstrap/{scope}/worker_bootstrap.json", - "bootstrap/{scope}/entity_graph.npz" - ], - "decorator_line": 198, - "description": "Filesystem path adapter for run-scoped local H5 bootstrap artifacts.", - "id": "local_h5_worker_bootstrap_store", - "label": "WorkerBootstrapStore", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/bootstrap.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "WorkerBootstrapStore", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_bootstrap.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db" - ], - "artifacts_out": [ - "bootstrap/{scope}/worker_bootstrap.json", - "bootstrap/{scope}/entity_graph.npz" - ], - "decorator_line": 259, - "description": "Materialize deterministic local H5 worker bootstrap artifacts.", - "id": "local_h5_worker_bootstrap_builder", - "label": "WorkerBootstrapBuilder", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/bootstrap.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "WorkerBootstrapBuilder", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_bootstrap.py" - ] - }, - { - "decorator_line": 71, - "description": "In-memory local H5 payload and diagnostics for one area.", - "id": "local_h5_build_result", - "label": "LocalAreaBuildResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/builder.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "LocalAreaBuildResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_builder.py" - ] - }, - { - "decorator_line": 139, - "description": "Build the in-memory payload for one local-area or national H5 output.", - "id": "local_h5_dataset_builder", - "label": "LocalAreaDatasetBuilder", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/builder.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "LocalAreaDatasetBuilder", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_builder.py" - ] - }, - { - "decorator_line": 31, - "description": "Input artifact and run metadata bundle for one local H5 publish scope.", - "id": "local_h5_publishing_input_bundle", - "label": "PublishingInputBundle", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "PublishingInputBundle" - }, - { - "decorator_line": 78, - "description": "Stable content identity for one local H5 publication input artifact.", - "id": "local_h5_artifact_identity", - "label": "ArtifactIdentity", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "stable", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "ArtifactIdentity", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "decorator_line": 113, - "description": "Provenance and resumability material for one local H5 publish scope.", - "id": "local_h5_traceability_bundle", - "label": "TraceabilityBundle", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "TraceabilityBundle", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "decorator_line": 188, - "description": "Build traceability bundles and deterministic scope fingerprints for local H5 publication.", - "id": "local_h5_traceability", - "label": "FingerprintingService", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "FingerprintingService", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "decorator_line": 40, - "description": "Resolved physical source used to recover exact calibration geography.", - "id": "local_h5_resolved_geography_source", - "label": "ResolvedGeographySource", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/geography_loader.py", - "stability": "stable", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "ResolvedGeographySource", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_geography_loader.py" - ] - }, - { - "decorator_line": 77, - "description": "Resolve and load saved, package-backed, or legacy calibration geography artifacts.", - "id": "calibration_geography_loader", - "label": "CalibrationGeographyLoader", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/geography_loader.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "CalibrationGeographyLoader", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_geography_loader.py" - ] - }, - { - "decorator_line": 29, - "description": "Assign weighted area work items to worker batches using longest-processing-time scheduling.", - "id": "local_h5_partition", - "label": "Partition Local H5 Work", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/partitioning.py", - "stability": "stable", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "partition_weighted_work_items", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_partitioning.py" - ] - }, - { - "decorator_line": 43, - "description": "Validated in-memory period-grouped payload for one local H5 output.", - "id": "local_h5_payload", - "label": "H5Payload", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/payload.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "H5Payload", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_payload.py" - ] - }, - { - "decorator_line": 122, - "description": "Context passed to payload postprocessors during one H5 build.", - "id": "local_h5_payload_build_context", - "label": "PayloadBuildContext", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/payload.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "PayloadBuildContext", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_payload.py" - ] - }, - { - "decorator_line": 20, - "description": "Reindex selected household, person, and subentity rows for local H5 outputs.", - "id": "local_h5_entity_reindexer", - "label": "EntityReindexer", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/reindexing.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "EntityReindexer", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_reindexing.py" - ] - }, - { - "decorator_line": 111, - "description": "Output entity IDs and source row indices after local H5 clone selection.", - "id": "local_h5_reindexed_entities", - "label": "ReindexedEntities", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/reindexing.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "ReindexedEntities", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_reindexing.py" - ] - }, - { - "decorator_line": 61, - "description": "Typed geography predicate for local H5 output selection.", - "id": "local_h5_area_filter", - "label": "AreaFilter", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/requests.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "AreaFilter", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_requests.py" - ] - }, - { - "decorator_line": 137, - "description": "Typed request contract for one national, state, district, city, or custom H5 output.", - "id": "local_h5_area_request", - "label": "AreaBuildRequest", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/requests.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "AreaBuildRequest", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_requests.py" - ] - }, - { - "decorator_line": 19, - "description": "Select active clone-household rows for one local H5 output.", - "id": "local_h5_area_selector", - "label": "AreaSelector", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/selection.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "AreaSelector", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_selection.py" - ] - }, - { - "decorator_line": 101, - "description": "Selected clone-household rows for one local H5 output.", - "id": "local_h5_clone_selection", - "label": "CloneSelection", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/selection.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "CloneSelection", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_selection.py" - ] - }, - { - "decorator_line": 63, - "description": "Explicit source-dataset entity spine for local H5 worker setup.", - "id": "local_h5_entity_graph", - "label": "EntityGraph", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "EntityGraph", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_source_dataset.py" - ] - }, - { - "decorator_line": 370, - "description": "Lazy source variable access wrapper for local H5 source snapshots.", - "id": "local_h5_microsimulation_variable_provider", - "label": "MicrosimulationVariableProvider", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "MicrosimulationVariableProvider", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_source_dataset.py" - ] - }, - { - "decorator_line": 469, - "description": "In-memory source H5 dataset contract for local H5 worker setup.", - "id": "local_h5_source_dataset_snapshot", - "label": "SourceDatasetSnapshot", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "SourceDatasetSnapshot", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_source_dataset.py" - ] - }, - { - "decorator_line": 546, - "description": "PolicyEngine H5 dataset adapter for local H5 source snapshots.", - "id": "local_h5_policyengine_dataset_reader", - "label": "PolicyEngineDatasetReader", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "PolicyEngineDatasetReader", - "validation_commands": [ - "uv run pytest tests/integration/build_outputs/h5_worker_runtime/test_worker_script_tiny_fixture.py" - ] - }, - { - "decorator_line": 54, - "description": "US entity ID and household-weight local H5 payload data.", - "id": "local_h5_us_entity_postprocessor_result", - "label": "USEntityPostProcessorResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USEntityPostProcessorResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 80, - "description": "US geography local H5 payload data and block-derived geography.", - "id": "local_h5_us_geography_postprocessor_result", - "label": "USGeographyPostProcessorResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USGeographyPostProcessorResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 107, - "description": "US take-up local H5 payload data and generated take-up variables.", - "id": "local_h5_us_takeup_postprocessor_result", - "label": "USTakeupPostProcessorResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USTakeupPostProcessorResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 134, - "description": "Apply US entity ID and household-weight fields to local H5 payloads.", - "id": "local_h5_us_entity_postprocessor", - "label": "USEntityPostProcessor", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USEntityPostProcessor", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 187, - "description": "Apply US geography fields to local H5 payloads.", - "id": "local_h5_us_geography_postprocessor", - "label": "USGeographyPostProcessor", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USGeographyPostProcessor", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 292, - "description": "Apply US take-up fields to local H5 payloads.", - "id": "local_h5_us_takeup_postprocessor", - "label": "USTakeupPostProcessor", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "USTakeupPostProcessor", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 23, - "description": "Worker-scoped local H5 validation policy contract.", - "id": "local_h5_validation_policy", - "label": "ValidationPolicy", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/validation.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "ValidationPolicy", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_validation.py" - ] - }, - { - "decorator_line": 41, - "description": "Prepared per-worker local H5 validation target context.", - "id": "local_h5_validation_context", - "label": "ValidationContext", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/validation.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "ValidationContext", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_validation.py" - ] - }, - { - "artifacts_in": [ - "policy_data.db", - "target_config.yaml", - "target_config_full.yaml" - ], - "decorator_line": 115, - "description": "Prepare local H5 validation targets once per worker session.", - "id": "local_h5_area_validation_service", - "label": "AreaValidationService", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/validation.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "AreaValidationService", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_validation.py" - ] - }, - { - "decorator_line": 38, - "description": "Clone selected source variables into period-grouped local H5 payloads.", - "id": "local_h5_variable_cloner", - "label": "VariableCloner", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/variables.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "VariableCloner", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_variables.py" - ] - }, - { - "decorator_line": 126, - "description": "Period-grouped cloned source variables for one local H5 output.", - "id": "local_h5_variable_clone_payload", - "label": "VariableClonePayload", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/variables.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "VariableClonePayload", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_variables.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "national_calibration_weights.npy" - ], - "decorator_line": 20, - "description": "Explicit shape contract for flat clone-level calibration weights.", - "id": "clone_weight_matrix", - "label": "CloneWeightMatrix", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/weights.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "CloneWeightMatrix", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_weights.py" - ] - }, - { - "decorator_line": 50, - "description": "Normalized worker-execution input payload for local H5 builds.", - "id": "local_h5_worker_calibration_inputs", - "label": "WorkerCalibrationInputs", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/worker_inputs.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "WorkerCalibrationInputs", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_worker_inputs.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db", - "worker_bootstrap.json" - ], - "decorator_line": 34, - "description": "Worker-scoped local H5 setup state reused across queued requests.", - "id": "local_h5_worker_session", - "label": "WorkerSession", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/worker_session.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "WorkerSession", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_worker_session.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db", - "bootstrap/{scope}/worker_bootstrap.json", - "bootstrap/{scope}/entity_graph.npz" - ], - "decorator_line": 69, - "description": "Load local H5 source, weights, geography, and validation context once per worker.", - "id": "local_h5_worker_session_factory", - "label": "WorkerSessionFactory", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/worker_session.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "WorkerSessionFactory", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_worker_session.py", - "uv run pytest tests/integration/build_outputs/h5_worker_runtime/test_worker_script_tiny_fixture.py" - ] - }, - { - "decorator_line": 18, - "description": "Post-write verification summary for one local H5 file.", - "id": "local_h5_write_result", - "label": "H5WriteResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/writer.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "H5WriteResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_writer.py" - ] - }, - { - "decorator_line": 40, - "description": "Write one period-grouped local H5 payload to disk.", - "id": "local_h5_writer", - "label": "H5Writer", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/writer.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "H5Writer", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_writer.py" - ] - }, - { - "decorator_line": 540, - "description": "Derive state, county, tract, PUMA, place, district, and other geography from block GEOIDs.", - "id": "geo_derive", - "label": "Derive Geography From Blocks", - "node_type": "library", - "pathways": [ - "local_h5", - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/block_assignment.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "derive_geography_from_blocks", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_clone_and_assign.py" - ] - }, - { - "artifacts_out": [ - "GeographyAssignment" - ], - "decorator_line": 147, - "description": "Assign cloned CPS records to population-weighted blocks, counties, states, and congressional districts.", - "id": "geo_assign", - "label": "Assign Random Geography", - "node_type": "library", - "pathways": [ - "data_build", - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/clone_and_assign.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "assign_random_geography", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_clone_and_assign.py" - ] - }, - { - "artifacts_in": [ - "extended_cps_2024.h5" - ], - "artifacts_out": [ - "stratified_extended_cps_2024.h5" - ], - "decorator_line": 68, - "description": "Create a calibration-sized stratified CPS sample while preserving high-AGI tail records.", - "id": "create_stratified", - "label": "Create Stratified CPS Dataset", - "node_type": "entrypoint", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/create_stratified_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "create_stratified_cps_dataset", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_create_stratified_cps.py" - ] - }, - { - "artifacts_in": [ - "local_area_build/**/*.h5" - ], - "decorator_line": 96, - "description": "Upload locally built H5 files into Hugging Face staging paths.", - "id": "local_stage_upload", - "label": "Stage Local H5 Files", - "node_type": "entrypoint", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/promote_local_h5s.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "stage", - "validation_commands": [ - "uv run pytest tests/unit/test_promote_local_h5s.py" - ] - }, - { - "artifacts_out": [ - "production H5 release", - "release manifest" - ], - "decorator_line": 116, - "description": "Promote staged H5 files to production storage and publish release manifests.", - "id": "atomic_promote", - "label": "Atomic Promote Local H5 Files", - "node_type": "entrypoint", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/promote_local_h5s.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "current", - "target_symbol": "promote", - "validation_commands": [ - "uv run pytest tests/unit/test_promote_local_h5s.py" - ] - }, - { - "api_refs": [ - "policyengine_us_data.build_outputs.fingerprinting.FingerprintingService" - ], - "decorator_line": 60, - "description": "Compute a scope fingerprint for local H5 checkpoint and resume decisions.", - "id": "local_h5_input_fingerprint", - "label": "Compute Local H5 Input Fingerprint", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "legacy", - "target_symbol": "compute_input_fingerprint", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "api_refs": [ - "policyengine_us_data.build_outputs.geography_loader.CalibrationGeographyLoader" - ], - "artifacts_in": [ - "calibration_weights.npy", - "geography_assignment.npz", - "stacked_blocks.npy" - ], - "decorator_line": 114, - "description": "Resolve exact geography from saved bundles, package metadata, or legacy block artifacts.", - "id": "load_calibration_geography", - "label": "Load Calibration Geography", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "legacy", - "target_symbol": "load_calibration_geography", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_geography_loader.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps*.h5" - ], - "artifacts_out": [ - "states/*.h5", - "districts/*.h5", - "cities/*.h5", - "US.h5" - ], - "decorator_line": 306, - "description": "Expand calibrated clone weights into local-area H5 datasets with geography and takeup updates.", - "details": "This is the main bundled H5 construction routine and remains a critical transitional waypoint.", - "id": "build_h5", - "label": "Build Local Area H5", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "transitional", - "target_symbol": "build_h5", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_publish_local_area.py", - "uv run pytest tests/integration/test_tiny_h5_pipeline.py" - ] - }, - { - "artifacts_out": [ - "states/*.h5" - ], - "decorator_line": 441, - "description": "Build state-level H5 files from calibrated weights and exact geography.", - "id": "build_states", - "label": "Build State H5 Files", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "current", - "target_symbol": "build_states" - }, - { - "artifacts_out": [ - "districts/*.h5" - ], - "decorator_line": 527, - "description": "Build congressional-district H5 files from calibrated weights and exact geography.", - "id": "build_districts", - "label": "Build District H5 Files", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "current", - "target_symbol": "build_districts" - }, - { - "artifacts_out": [ - "cities/*.h5" - ], - "decorator_line": 614, - "description": "Build supported city H5 files with county probability filtering.", - "id": "build_cities", - "label": "Build City H5 Files", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "current", - "target_symbol": "build_cities" - }, - { - "decorator_line": 369, - "description": "Scale imputed Social Security subcomponents so they reconcile to total Social Security.", - "id": "ss_reconcile", - "label": "Social Security Subcomponent Reconciliation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "reconcile_ss_subcomponents", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "artifacts_out": [ - "extended_cps_2024.h5" - ], - "decorator_line": 436, - "description": "Double CPS records and populate the clone half with PUF-imputed tax variables.", - "id": "record_double", - "label": "PUF Clone Dataset", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "puf_clone_dataset", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 622, - "description": "Impute weeks unemployed for the clone half using CPS donor relationships.", - "id": "weeks_impute", - "label": "Weeks Unemployed Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "_impute_weeks_unemployed", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 730, - "description": "Impute and constrain retirement contribution inputs on the PUF clone half.", - "id": "retire_impute", - "label": "Retirement Contribution Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "_impute_retirement_contributions", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 875, - "description": "Run Quantile Random Forest imputation from PUF tax variables onto CPS clone records.", - "id": "puf_qrf_pass", - "label": "PUF QRF Imputation Pass", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "_run_qrf_imputation", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 312, - "description": "Check calibrated H5 structure, weights, IDs, mappings, takeup, and aggregate sanity.", - "id": "sanity_checks", - "label": "Run H5 Sanity Checks", - "node_type": "validation", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/sanity_checks.py", - "stability": "moving", - "stage_id": "4_build_outputs", - "status": "current", - "target_symbol": "run_sanity_checks", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_validate_staging.py" - ] - }, - { - "artifacts_in": [ - "stratified_extended_cps_2024.h5" - ], - "artifacts_out": [ - "source_imputed_stratified_extended_cps_2024.h5" - ], - "decorator_line": 170, - "description": "Apply ACS, SIPP, ORG, and SCF donor imputations to the stratified CPS calibration input.", - "id": "source_impute", - "label": "Source-Impute Stratified CPS", - "node_type": "entrypoint", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "impute_source_variables", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 319, - "description": "Impute rent and real estate tax variables from ACS donor data.", - "id": "acs_qrf", - "label": "ACS QRF Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "_impute_acs", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 420, - "description": "Impute tips, liquid assets, and vehicle assets from SIPP donor data.", - "id": "sipp_qrf", - "label": "SIPP QRF Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "_impute_sipp", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 804, - "description": "Impute net worth and auto-loan variables from SCF donor data.", - "id": "scf_qrf", - "label": "SCF QRF Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "_impute_scf", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "artifacts_in": [ - "calibration_package.pkl" - ], - "decorator_line": 696, - "description": "Build population-proportional initial weights from geography and targets.", - "id": "init_weights", - "label": "Compute Initial Weights", - "node_type": "library", - "pathways": [ - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "stage_id": "3_fit_weights", - "status": "current", - "target_symbol": "compute_initial_weights", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "artifacts_in": [ - "calibration_package.pkl" - ], - "artifacts_out": [ - "calibration_weights.npy", - "unified_diagnostics.csv" - ], - "decorator_line": 774, - "description": "Optimize sparse calibration weights with HardConcrete gates and diagnostics.", - "id": "fit_model", - "label": "Fit L0 Calibration Weights", - "node_type": "library", - "pathways": [ - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "stage_id": "3_fit_weights", - "status": "current", - "target_symbol": "fit_l0_weights", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "artifacts_out": [ - "unified_diagnostics.csv" - ], - "decorator_line": 1131, - "description": "Compare fitted weighted sums to calibration targets and summarize error.", - "id": "calibration_diagnostics", - "label": "Compute Calibration Diagnostics", - "node_type": "library", - "pathways": [ - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "stage_id": "3_fit_weights", - "status": "current", - "target_symbol": "compute_diagnostics", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "artifacts_in": [ - "source_imputed_stratified_extended_cps*.h5", - "policy_data.db" - ], - "artifacts_out": [ - "calibration_package.pkl", - "calibration_weights.npy", - "unified_diagnostics.csv", - "unified_run_config.json" - ], - "decorator_line": 1250, - "description": "Build or load the calibration package, fit sparse weights, and write diagnostics.", - "details": "This remains a bundled orchestration function; decorators document the semantic pathway until the implementation is further decomposed.", - "id": "run_calibration", - "label": "Run Unified Calibration", - "node_type": "entrypoint", - "pathways": [ - "calibration_package", - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "stage_id": "2_build_calibration_package", - "status": "transitional", - "target_symbol": "run_calibration", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "decorator_line": 215, - "description": "Run state-specific Microsimulations used to assemble calibration matrix columns.", - "id": "state_precomp", - "label": "Per-State Simulation Precomputation", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "stage_id": "2_build_calibration_package", - "status": "current", - "target_symbol": "_compute_single_state", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py" - ] - }, - { - "artifacts_out": [ - "sparse calibration COO/CSR shards" - ], - "decorator_line": 594, - "description": "Assemble target values for clone-household columns across assigned geographies.", - "id": "clone_assembly", - "label": "Clone Value Assembly", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "stage_id": "2_build_calibration_package", - "status": "current", - "target_symbol": "_assemble_clone_values_standalone", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py", - "uv run pytest tests/integration/test_chunked_matrix_builder.py" - ] - }, - { - "decorator_line": 1367, - "description": "Database-backed sparse matrix builder for national and local-area calibration.", - "id": "unified_matrix_builder", - "label": "UnifiedMatrixBuilder", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "stage_id": "2_build_calibration_package", - "status": "current", - "target_symbol": "UnifiedMatrixBuilder", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py" - ] - }, - { - "artifacts_out": [ - "X_sparse", - "targets_df", - "target_names" - ], - "decorator_line": 2465, - "description": "Build the in-memory sparse matrix for calibration targets and clone households.", - "id": "build_matrix", - "label": "Build Calibration Matrix", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "stage_id": "2_build_calibration_package", - "status": "current", - "target_symbol": "build_matrix", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py" - ] - }, - { - "artifacts_out": [ - "chunked COO shards", - "X_sparse" - ], - "decorator_line": 3202, - "description": "Stream matrix construction through clone-household chunks with resumable shard caches.", - "id": "build_matrix_chunked", - "label": "Build Calibration Matrix In Chunks", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "experimental", - "stage_id": "2_build_calibration_package", - "status": "current", - "target_symbol": "build_matrix_chunked", - "validation_commands": [ - "uv run pytest tests/integration/test_chunked_matrix_builder.py" - ] - }, - { - "decorator_line": 302, - "description": "Run microsimulation target comparisons for one staged area.", - "id": "target_validation", - "label": "Validate Area Against Targets", - "node_type": "validation", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/validate_staging.py", - "stability": "moving", - "stage_id": "5_validate_and_promote_release", - "status": "current", - "target_symbol": "validate_area", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_validate_staging.py" - ] - }, - { - "decorator_line": 339, - "description": "Impute rent and real estate taxes using ACS donor data.", - "id": "add_rent", - "label": "Rent Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "legacy", - "target_symbol": "add_rent", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 463, - "description": "Apply stochastic takeup and reported-anchor alignment for benefit programs.", - "id": "add_takeup", - "label": "Benefit Takeup", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_takeup", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 918, - "description": "Create person, household, tax-unit, SPM-unit, family, and marital-unit IDs.", - "id": "add_id_variables", - "label": "Add ID Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "stable", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_id_variables", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_cps_identification.py" - ] - }, - { - "decorator_line": 982, - "description": "Populate CPS personal demographics and occupation-derived inputs.", - "id": "add_personal_variables", - "label": "Add Personal Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_personal_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1126, - "description": "Populate CPS income, transfer, retirement, and QBI-related personal inputs.", - "id": "add_personal_income_variables", - "label": "Add Income Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_personal_income_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1402, - "description": "Populate CPS supplemental poverty measure variables.", - "id": "add_spm_variables", - "label": "Add SPM Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_spm_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1441, - "description": "Populate household geography variables including state, county, and NYC flag.", - "id": "add_household_variables", - "label": "Add Household Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "stable", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_household_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1483, - "description": "Link CPS records across adjacent years and populate prior-year income inputs.", - "id": "add_previous_year_income", - "label": "Previous-Year Income", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_previous_year_income", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1589, - "description": "Classify SSN card type and immigration-related CPS inputs from ASEC conditions.", - "id": "add_ssn_card_type", - "label": "Add SSN Card Type", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_ssn_card_type", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 2488, - "description": "Impute tip income and household asset inputs from SIPP donor data.", - "id": "add_tips", - "label": "Tips And Asset Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "legacy", - "target_symbol": "add_tips", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 2663, - "description": "Impute hourly wage, hourly-pay status, and union coverage from CPS ORG donors.", - "id": "add_org_inputs", - "label": "ORG Labor-Market Inputs", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "add_org_labor_market_inputs", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_org.py" - ] - }, - { - "decorator_line": 2779, - "description": "Impute auto loan balance, auto loan interest, and net worth from SCF donor data.", - "id": "add_auto_loan", - "label": "Auto Loan And Net Worth Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "legacy", - "target_symbol": "add_auto_loan_interest_and_net_worth", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 306, - "description": "Subsample CPS arrays for released CPS vintages while full variants skip this step.", - "id": "downsample", - "label": "Downsample CPS", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "stable", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "downsample", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "artifacts_in": [ - "extended_cps_2024" - ], - "artifacts_out": [ - "aca_2025_takeup" - ], - "decorator_line": 404, - "description": "Adds synthetic 2025 ACA take-up assignments until calibrated person-level APTC enrollment reaches the target.", - "id": "aca_2025_override", - "label": "ACA 2025 Take-Up Override", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/enhanced_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "transitional", - "target_symbol": "create_aca_2025_takeup_override" - }, - { - "artifacts_in": [ - "loss_matrix", - "calibration_targets" - ], - "artifacts_out": [ - "enhanced_cps_weights" - ], - "decorator_line": 487, - "description": "Fits enhanced CPS weights against calibration targets with the hard-concrete loss machinery.", - "id": "reweight", - "label": "Enhanced CPS Reweighting", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/enhanced_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "transitional", - "target_symbol": "reweight" - }, - { - "artifacts_in": [ - "qrf_pass2", - "record_double" - ], - "artifacts_out": [ - "clone_feature_splice" - ], - "decorator_line": 382, - "description": "Replaces clone-half CPS feature variables with donor-matched predictions so doubled records retain plausible demographics and occupation labels.", - "id": "clone_features", - "label": "Splice Clone Features", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "transitional", - "target_symbol": "_splice_clone_feature_predictions" - }, - { - "artifacts_in": [ - "record_double", - "preprocess_cps" - ], - "artifacts_out": [ - "cps_only_predictions" - ], - "decorator_line": 422, - "description": "Runs the second-stage CPS-only QRF imputation for PUF clone records inside the extended CPS build.", - "id": "cps_only", - "label": "Impute CPS-Only Variables", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "transitional", - "target_symbol": "_impute_cps_only_variables" - }, - { - "artifacts_in": [ - "cps_only_predictions" - ], - "artifacts_out": [ - "extended_cps_stage2" - ], - "decorator_line": 700, - "description": "Writes second-stage CPS-only QRF predictions back into the PUF clone half of the extended CPS record set.", - "id": "qrf_pass2", - "label": "Splice CPS-Only Predictions", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "transitional", - "target_symbol": "_splice_cps_only_predictions" - }, - { - "artifacts_in": [ - "extended_cps_stage2" - ], - "artifacts_out": [ - "formula_pruned_extended_cps" - ], - "decorator_line": 1179, - "description": "Removes variables computed by policyengine-us formulas, while preserving selected imputed inputs under canonical leaf variable names.", - "id": "formula_drop", - "label": "Drop Formula Variables", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "transitional", - "target_symbol": "_drop_formula_variables" - }, - { - "decorator_line": 301, - "description": "Simulate Section 199A W-2 wage and UBIA guardrail quantities from PUF income.", - "id": "simulate_qbi", - "label": "QBI Simulation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "simulate_w2_and_ubia_from_puf", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 405, - "description": "Impute pre-tax retirement contributions onto PUF tax units from CPS donors.", - "id": "impute_puf_pension", - "label": "Impute PUF Pension Contributions", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "impute_pension_contributions_to_puf", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 461, - "description": "Impute missing PUF demographics from demographic donor records.", - "id": "impute_puf_demographics", - "label": "Impute PUF Demographics", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "impute_missing_demographics", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 620, - "description": "Rename IRS variables and derive PolicyEngine-ready PUF tax inputs.", - "id": "preprocess_puf", - "label": "Preprocess PUF", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "preprocess_puf", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 45, - "description": "Impute SCF-backed tax-unit mortgage balance hints for structural mortgage conversion.", - "id": "mortgage_hints", - "label": "Mortgage Balance Hint Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/utils/mortgage_interest.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "impute_tax_unit_mortgage_balance_hints", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_mortgage_interest.py" - ] - }, - { - "decorator_line": 126, - "description": "Convert deductible mortgage interest into structural mortgage balances, interest, and origination-year inputs.", - "id": "mortgage_convert", - "label": "Structural Mortgage Conversion", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/utils/mortgage_interest.py", - "stability": "moving", - "stage_id": "1_build_datasets", - "status": "current", - "target_symbol": "convert_mortgage_interest_to_structural_inputs", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_mortgage_interest.py" - ] - } - ], - "stats": { - "by_pathway": { - "calibration_package": 8, - "data_build": 36, - "local_h5": 53, - "weight_fit": 4 - }, - "by_stage": { - "1_build_datasets": 36, - "2_build_calibration_package": 6, - "3_fit_weights": 3, - "4_build_outputs": 45, - "5_validate_and_promote_release": 8 - }, - "by_status": { - "current": 85, - "legacy": 5, - "transitional": 8 - }, - "by_type": { - "entrypoint": 5, - "library": 85, - "process": 6, - "validation": 2 - }, - "edge_count": 25, - "node_count": 98 - }, - "unproduced_artifacts": [ - "calibration_targets", - "extended_cps_2024", - "geography_assignment.npz", - "local_area_build/**/*.h5", - "loss_matrix", - "national_calibration_weights.npy", - "policy_data.db", - "preprocess_cps", - "qrf_pass2", - "record_double", - "source_imputed_stratified_extended_cps*.h5", - "source_imputed_stratified_extended_cps.h5", - "stacked_blocks.npy", - "target_config.yaml", - "target_config_full.yaml" - ] -} \ No newline at end of file diff --git a/backend/data/pipeline/stages/calibration_package.input.json b/backend/data/pipeline/stages/calibration_package.input.json deleted file mode 100644 index 40c287d..0000000 --- a/backend/data/pipeline/stages/calibration_package.input.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "stage_id": "calibration_package", - "label": "calibration_package", - "nodes": [ - { - "decorator_line": 540, - "description": "Derive state, county, tract, PUMA, place, district, and other geography from block GEOIDs.", - "id": "geo_derive", - "label": "Derive Geography From Blocks", - "node_type": "library", - "pathways": [ - "local_h5", - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/block_assignment.py", - "stability": "moving", - "status": "current", - "target_symbol": "derive_geography_from_blocks", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_clone_and_assign.py" - ] - }, - { - "artifacts_out": [ - "GeographyAssignment" - ], - "decorator_line": 147, - "description": "Assign cloned CPS records to population-weighted blocks, counties, states, and congressional districts.", - "id": "geo_assign", - "label": "Assign Random Geography", - "node_type": "library", - "pathways": [ - "data_build", - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/clone_and_assign.py", - "stability": "moving", - "status": "current", - "target_symbol": "assign_random_geography", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_clone_and_assign.py" - ] - }, - { - "artifacts_in": [ - "source_imputed_stratified_extended_cps*.h5", - "policy_data.db" - ], - "artifacts_out": [ - "calibration_package.pkl", - "calibration_weights.npy", - "unified_diagnostics.csv", - "unified_run_config.json" - ], - "decorator_line": 1250, - "description": "Build or load the calibration package, fit sparse weights, and write diagnostics.", - "details": "This remains a bundled orchestration function; decorators document the semantic pathway until the implementation is further decomposed.", - "id": "run_calibration", - "label": "Run Unified Calibration", - "node_type": "entrypoint", - "pathways": [ - "calibration_package", - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "run_calibration", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "decorator_line": 215, - "description": "Run state-specific Microsimulations used to assemble calibration matrix columns.", - "id": "state_precomp", - "label": "Per-State Simulation Precomputation", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "status": "current", - "target_symbol": "_compute_single_state", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py" - ] - }, - { - "artifacts_out": [ - "sparse calibration COO/CSR shards" - ], - "decorator_line": 594, - "description": "Assemble target values for clone-household columns across assigned geographies.", - "id": "clone_assembly", - "label": "Clone Value Assembly", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "status": "current", - "target_symbol": "_assemble_clone_values_standalone", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py", - "uv run pytest tests/integration/test_chunked_matrix_builder.py" - ] - }, - { - "decorator_line": 1367, - "description": "Database-backed sparse matrix builder for national and local-area calibration.", - "id": "unified_matrix_builder", - "label": "UnifiedMatrixBuilder", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "status": "current", - "target_symbol": "UnifiedMatrixBuilder", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py" - ] - }, - { - "artifacts_out": [ - "X_sparse", - "targets_df", - "target_names" - ], - "decorator_line": 2465, - "description": "Build the in-memory sparse matrix for calibration targets and clone households.", - "id": "build_matrix", - "label": "Build Calibration Matrix", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "moving", - "status": "current", - "target_symbol": "build_matrix", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_matrix_builder.py" - ] - }, - { - "artifacts_out": [ - "chunked COO shards", - "X_sparse" - ], - "decorator_line": 3202, - "description": "Stream matrix construction through clone-household chunks with resumable shard caches.", - "id": "build_matrix_chunked", - "label": "Build Calibration Matrix In Chunks", - "node_type": "library", - "pathways": [ - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/unified_matrix_builder.py", - "stability": "experimental", - "status": "current", - "target_symbol": "build_matrix_chunked", - "validation_commands": [ - "uv run pytest tests/integration/test_chunked_matrix_builder.py" - ] - } - ] -} \ No newline at end of file diff --git a/backend/data/pipeline/stages/calibration_package.md b/backend/data/pipeline/stages/calibration_package.md deleted file mode 100644 index c193d59..0000000 --- a/backend/data/pipeline/stages/calibration_package.md +++ /dev/null @@ -1,68 +0,0 @@ -# Calibration package build pathway - -## Purpose - -This stage produces the **calibration package**: the sparse matrix `X` (shape `n_targets x (n_records * n_clones)`), the aligned `targets_df` of right-hand-side values, and the metadata that downstream weight-fitting needs. Each column of `X` is one clone of one base CPS household placed at a specific census block / county / state / congressional district, and each row is one administrative target (e.g. "EITC dollars in NY-17 for joint filers with two children"). The cell `X[i, j]` is the contribution of household-column `j` to target-row `i` under the constraints that define that target. The optimizer then solves for a non-negative weight vector `w` such that `X @ w ≈ targets`, so this package is the entire problem statement the L0/L2 solver consumes. - -## Inputs and outputs - -**Inputs** -- `dataset_path`: an enhanced CPS h5 (`source_imputed_stratified_extended_cps*.h5`) — the base microdata, one record per household with person/tax_unit/spm_unit children. -- `db_path` → `policy_data.db` (SQLite): provides `target_overview` rows (geographic_id, variable, period, value, stratum_id) plus per-stratum constraints. Queried via `UnifiedMatrixBuilder._query_targets` and `_get_stratum_constraints`. -- Block-level Census distribution (population + AGI weights, loaded by `load_global_block_distribution`) used to draw geographies. -- Optional `target_config` YAML to filter the target set; optional `fixed_state_fips` from Forbes synthetic records; optional CD-AGI targets for AGI-conditional block draws. - -**Outputs (the `calibration_package.pkl` payload)** -- `X_sparse`: `scipy.sparse.csr_matrix(shape=(n_targets, n_records*n_clones), dtype=float32)` — the calibration matrix. -- `targets_df`: DataFrame of selected targets, sorted by `(_geo_level, variable, reform_id, geographic_id)`, with `value` already uprated to `time_period` (`original_value`, `uprating_factor` columns kept for provenance). -- `target_names`: list of human-readable names produced by `_make_target_name(variable, constraints, reform_id)`, one per row of `X_sparse`. -- `initial_weights`: population-proportional starting vector from `compute_initial_weights` (uses person_count targets per CD). -- `cd_geoid`, `block_geoid`: geography arrays of length `n_records*n_clones` for the columns. -- `metadata`: dataset/db SHA256 checksums, git provenance, `n_clones`, `seed`, `package_scope`, `matrix_builder` mode (`chunked` vs `precompute`), `created_at`. - -Side artifacts in the same `run_calibration` orchestration: `calibration_weights.npy`, `unified_diagnostics.csv`, `unified_run_config.json` (these belong to the weight-fit stage that consumes the package). - -## Key sub-stages (ordered) - -1. **`geo_derive`** (`block_assignment.derive_geography_from_blocks`, line 540): given an array of 15-char block GEOIDs, derive county FIPS, tract GEOID, state FIPS, CBSA, SLDU/SLDL, place, VTD, PUMA, ZCTA, and the county enum index. Used both by direct local-area pipelines and by downstream consumers reading already-assigned blocks. -2. **`geo_assign`** (`clone_and_assign.assign_random_geography`, line 147): produces the `GeographyAssignment` for `n_records * n_clones` columns. Clone 0 is an unrestricted population-weighted (optionally AGI-weighted for AGI≥90th-pctile households) draw from the global block distribution; clones 1..N redraw collisions up to 50 times to spread each base record across distinct CDs (`used_cd_by_record` tracking at line 302). Honors `fixed_state_fips` to pin Forbes synthetic households. -3. **`state_precomp`** (`unified_matrix_builder._compute_single_state`, line 215): for each of the ~51 unique state FIPS in the assignment, spin up a fresh `Microsimulation`, set `state_fips` to that state for every record, clear cached arrays, and compute (a) household-level target values, (b) entity-level (`tax_unit`/`spm_unit`/`person`) target values, (c) person-level constraint variables, and optionally (d) reform-side income tax and (e) entity-level "eligible-if-takeup" values for takeup-affected targets and a `would_file_taxes_voluntarily=False` variant. Picklable top-level function so `ProcessPoolExecutor` can parallelize over states. -4. **County precomputation** (`_build_county_values` / `_compute_single_state_group_counties`, line 416): a smaller pass for `COUNTY_DEPENDENT_VARS` (e.g. `aca_ptc`) only, grouping counties by state and reusing one sim per state. -5. **`clone_assembly`** (`_assemble_clone_values_standalone`, line 594): per clone, slices the precomputed per-state arrays by each column's assigned state (and county, for county-dependent vars), producing `(hh_vars, person_vars, reform_hh_vars)` aligned to the clone's record order. Also `_assemble_target_entity_values_standalone` for non-household entity arrays. Raises `ValueError` if a county-dependent var is missing without `allow_state_fallback_for_county_dependent_targets`. -6. **`build_matrix`** (`UnifiedMatrixBuilder.build_matrix`, line 2465): the main orchestrator. (1) queries + uprates targets, (2) sorts by geo level, (3) precomputes constraints/target_names, (4) calls state and county precomp, (5) loops over `n_clones`: assembles clone values, applies geo-salted block-level takeup (`compute_block_takeup_for_entities`) for ACA/Medicaid/voluntary-filing, evaluates each target's constraints via `_calculate_target_values_standalone` (with a `(variable, constraint_key, reform_id)` cache to avoid recomputing identical strata), writes the nonzero entries to a per-clone COO `.npz` shard (`clone_{ci:04d}.npz`) or in-memory list. (6) Concatenates all COO triples into a single `csr_matrix`. Has both sequential and `ProcessPoolExecutor` paths sharing data via `_init_clone_worker`. -7. **`build_matrix_chunked`** (`UnifiedMatrixBuilder.build_matrix_chunked`, line 3202): experimental alternative. Instead of looping `n_clones` × all-records, it slices the full `n_total` column space into fixed-size chunks (default 25k columns), materializes a sliced H5 per chunk, simulates the chunk's mixed-geography records once, and writes a COO shard per chunk into `chunk_root/coo/chunk_NNNNNN.npz`. Chunks are resumable via a manifest signature (`build_chunk_lineage_signature` / `_validate_chunk_manifest`), and the final CSR is stream-assembled by `stream_csr_from_shards` in two passes (one for `indptr`, one to scatter). Supports Modal fan-out (`dispatch_chunks_modal`) for distributed builds. -8. **`unified_matrix_builder`** (the `UnifiedMatrixBuilder` class itself, line 1367): the SQLite-backed container that owns the engine, the `_query_targets` view selection, uprating (`_calculate_uprating_factors`, `_apply_hierarchical_uprating`), constraint resolution (`_get_stratum_constraints`), and the `_entity_rel_cache`. Both `build_matrix` and `build_matrix_chunked` are methods on it. - -`run_calibration` (`unified_calibration.run_calibration`, line 1250) is the entrypoint that glues steps 2 → 8 together, calls `compute_initial_weights`, and writes the pickle via `save_calibration_package`. - -## Common failure modes - -- **Missing county-level values for county-dependent targets**: `_assemble_clone_values_standalone` raises `ValueError("Missing county-level household values for county-dependent target ...")` when a target is in `COUNTY_DEPENDENT_VARS`, the geography lands in a county whose precompute is missing, and `allow_state_fallback_for_county_dependent_targets=False`. Triggered when `county_level=True` but the county precomp pass crashed or skipped that county. Workaround: pass `--skip-county` to fall back to state values. -- **Per-state simulation failure**: in the parallel state-precomp path (line 1542), any worker exception aborts the whole job (`RuntimeError(f"State {st} failed: ...")`) and cancels remaining futures. The same pattern applies to the clone-loop worker (line 2837). The actual variable-level failures inside `_compute_single_state` are swallowed as `logger.warning` — a target var that fails everywhere will silently be zero in `X`, which the optimizer then cannot achieve. -- **Empty target filter**: `build_matrix` (line 2532) and `build_matrix_chunked` (line 3263) both raise `"No targets found matching filter"` when `target_filter`/`target_config` excludes everything. -- **Stale takeup anchors**: when `rerandomize_takeup=True`, the builder reads `reported_has_subsidized_marketplace_health_coverage_at_interview` and `has_medicaid_health_coverage_at_interview` from the source h5 at `time_period`. If the period key is missing, the anchor is silently skipped (line 2672), causing block-level takeup draws to be unconstrained. -- **Chunk-cache lineage mismatch**: `build_matrix_chunked` with `resume_chunks=True` calls `_validate_chunk_manifest` against `build_chunk_lineage_signature`. Any change to dataset SHA, db SHA, geography, targets, chunk_size, or `rerandomize_takeup` invalidates the manifest and raises before reusing stale shards. -- **Constraint variable not precomputed**: `_evaluate_person_constraints_standalone` warns `"Constraint var '%s' not in precomputed person_vars"` and returns all-False, which zeros out every row using that stratum. Symptom in `targets_df`: many targets with `row_sum == 0`, dropped from `achievable` in the fit step. - -## Why this exists / design notes - -- **Per-state precompute first, clone loop second.** The naive approach — call `Microsimulation.calculate` once per clone — would re-run state tax/benefit logic billions of times. Instead, the builder runs one fresh sim per unique state FIPS (line 1437: "Creates a fresh Microsimulation per state to prevent cross-state cache pollution") and stores arrays indexed by base-record position. The clone loop then just slices those arrays by each column's assigned state. This turns N_clones × N_states sims into N_states sims plus cheap array slicing. -- **Fresh sim per state.** PolicyEngine's `Microsimulation` aggressively caches intermediates; switching `state_fips` mid-simulation leaves stale state-specific intermediates (e.g. state tax parameters) attached to other variables. The builder explicitly does not reuse a sim across states; it does reuse within a state across counties (line 1741: "within-state recalculation is clean"). -- **COO → CSR two-phase build.** Each clone writes a small `.npz` of `(rows, cols, vals)` triples to `cache_dir`. Assembly concatenates them and constructs the CSR once. This (a) bounds peak memory per clone, (b) makes the clone loop resumable (cached clones are skipped at line 2805), and (c) parallelizes cleanly since workers only need to write disjoint files. The chunked variant pushes this further with `stream_csr_from_shards`: two passes over shards to build `indptr` then scatter, never holding all triples in memory at once. -- **Constraint result caching by `(variable, constraint_key, reform_id)`.** Within a single clone, many targets share the same stratum (e.g. all CD-level "EITC for joint filers with 2 kids" rows). `target_value_cache` (line 3059) computes the per-record value vector once per stratum-key per clone, then slices it for each row's geo-restricted column set. -- **Two builders, one contract.** `build_matrix` (in-memory, fast, default) and `build_matrix_chunked` (streamed, resumable, Modal-capable) both return the same `(targets_df, X_csr, target_names)` tuple. Chunked exists for the local-area pathway where `n_total` grows past memory limits or where a long build must survive interruption. -- **Takeup re-randomization happens in the clone loop, not the precompute.** Block-level salting (`compute_block_takeup_for_entities`) needs the assigned `block_geoid` per column, which only exists at clone-loop time. State precompute stores the "eligible-if-takeup" entity-level amount; the clone loop multiplies by per-block boolean draws and re-aggregates back to household level (lines 3035–3057). -- **Initial weights are not optimization output.** `compute_initial_weights` produces a population-proportional starting point per CD from age-bucket person_count targets; it lives in the package so the fit stage has a deterministic warm start. - -## Source map - -| Node id | Source file | Role | -|---|---|---| -| `geo_derive` | `policyengine_us_data/calibration/block_assignment.py:540` | Map block GEOID → county/tract/state/CBSA/SLDU/SLDL/place/VTD/PUMA/ZCTA | -| `geo_assign` | `policyengine_us_data/calibration/clone_and_assign.py:147` | Draw blocks (pop- or AGI-weighted) for each of `n_records * n_clones` columns, spreading clones across distinct CDs | -| `state_precomp` | `policyengine_us_data/calibration/unified_matrix_builder.py:215` | Per-state fresh-sim precompute of hh/person/entity target & constraint values (picklable for `ProcessPoolExecutor`) | -| `clone_assembly` | `policyengine_us_data/calibration/unified_matrix_builder.py:594` | Slice per-state precomputed arrays by each clone's assigned state/county into clone-aligned vectors | -| `unified_matrix_builder` | `policyengine_us_data/calibration/unified_matrix_builder.py:1367` | DB-backed builder class: target querying, uprating, constraint resolution, entity-rel cache | -| `build_matrix` | `policyengine_us_data/calibration/unified_matrix_builder.py:2465` | In-memory orchestrator: precompute → clone loop → COO shards → final CSR | -| `build_matrix_chunked` | `policyengine_us_data/calibration/unified_matrix_builder.py:3202` | Streamed/resumable variant: fixed-size column chunks, per-chunk H5 + COO shards, manifest-validated resume, optional Modal fan-out | -| `run_calibration` | `policyengine_us_data/calibration/unified_calibration.py:1250` | Entrypoint orchestrating geo assignment → matrix build → `save_calibration_package` → fit (bundled) | diff --git a/backend/data/pipeline/stages/data_build.input.json b/backend/data/pipeline/stages/data_build.input.json deleted file mode 100644 index b1a2d97..0000000 --- a/backend/data/pipeline/stages/data_build.input.json +++ /dev/null @@ -1,664 +0,0 @@ -{ - "stage_id": "data_build", - "label": "data_build", - "nodes": [ - { - "artifacts_out": [ - "GeographyAssignment" - ], - "decorator_line": 147, - "description": "Assign cloned CPS records to population-weighted blocks, counties, states, and congressional districts.", - "id": "geo_assign", - "label": "Assign Random Geography", - "node_type": "library", - "pathways": [ - "data_build", - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/clone_and_assign.py", - "stability": "moving", - "status": "current", - "target_symbol": "assign_random_geography", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_clone_and_assign.py" - ] - }, - { - "artifacts_in": [ - "extended_cps_2024.h5" - ], - "artifacts_out": [ - "stratified_extended_cps_2024.h5" - ], - "decorator_line": 68, - "description": "Create a calibration-sized stratified CPS sample while preserving high-AGI tail records.", - "id": "create_stratified", - "label": "Create Stratified CPS Dataset", - "node_type": "entrypoint", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/create_stratified_cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "create_stratified_cps_dataset", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_create_stratified_cps.py" - ] - }, - { - "decorator_line": 369, - "description": "Scale imputed Social Security subcomponents so they reconcile to total Social Security.", - "id": "ss_reconcile", - "label": "Social Security Subcomponent Reconciliation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "reconcile_ss_subcomponents", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "artifacts_out": [ - "extended_cps_2024.h5" - ], - "decorator_line": 436, - "description": "Double CPS records and populate the clone half with PUF-imputed tax variables.", - "id": "record_double", - "label": "PUF Clone Dataset", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "puf_clone_dataset", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 622, - "description": "Impute weeks unemployed for the clone half using CPS donor relationships.", - "id": "weeks_impute", - "label": "Weeks Unemployed Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "_impute_weeks_unemployed", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 730, - "description": "Impute and constrain retirement contribution inputs on the PUF clone half.", - "id": "retire_impute", - "label": "Retirement Contribution Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "_impute_retirement_contributions", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "decorator_line": 875, - "description": "Run Quantile Random Forest imputation from PUF tax variables onto CPS clone records.", - "id": "puf_qrf_pass", - "label": "PUF QRF Imputation Pass", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/puf_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "_run_qrf_imputation", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_calibration_puf_impute.py" - ] - }, - { - "artifacts_in": [ - "stratified_extended_cps_2024.h5" - ], - "artifacts_out": [ - "source_imputed_stratified_extended_cps_2024.h5" - ], - "decorator_line": 170, - "description": "Apply ACS, SIPP, ORG, and SCF donor imputations to the stratified CPS calibration input.", - "id": "source_impute", - "label": "Source-Impute Stratified CPS", - "node_type": "entrypoint", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "impute_source_variables", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 319, - "description": "Impute rent and real estate tax variables from ACS donor data.", - "id": "acs_qrf", - "label": "ACS QRF Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "_impute_acs", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 420, - "description": "Impute tips, liquid assets, and vehicle assets from SIPP donor data.", - "id": "sipp_qrf", - "label": "SIPP QRF Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "_impute_sipp", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 804, - "description": "Impute net worth and auto-loan variables from SCF donor data.", - "id": "scf_qrf", - "label": "SCF QRF Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/calibration/source_impute.py", - "stability": "moving", - "status": "current", - "target_symbol": "_impute_scf", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_source_impute.py" - ] - }, - { - "decorator_line": 339, - "description": "Impute rent and real estate taxes using ACS donor data.", - "id": "add_rent", - "label": "Rent Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "legacy", - "target_symbol": "add_rent", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 463, - "description": "Apply stochastic takeup and reported-anchor alignment for benefit programs.", - "id": "add_takeup", - "label": "Benefit Takeup", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_takeup", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 918, - "description": "Create person, household, tax-unit, SPM-unit, family, and marital-unit IDs.", - "id": "add_id_variables", - "label": "Add ID Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "stable", - "status": "current", - "target_symbol": "add_id_variables", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_cps_identification.py" - ] - }, - { - "decorator_line": 982, - "description": "Populate CPS personal demographics and occupation-derived inputs.", - "id": "add_personal_variables", - "label": "Add Personal Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_personal_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1126, - "description": "Populate CPS income, transfer, retirement, and QBI-related personal inputs.", - "id": "add_personal_income_variables", - "label": "Add Income Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_personal_income_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1402, - "description": "Populate CPS supplemental poverty measure variables.", - "id": "add_spm_variables", - "label": "Add SPM Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_spm_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1441, - "description": "Populate household geography variables including state, county, and NYC flag.", - "id": "add_household_variables", - "label": "Add Household Variables", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "stable", - "status": "current", - "target_symbol": "add_household_variables", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1483, - "description": "Link CPS records across adjacent years and populate prior-year income inputs.", - "id": "add_previous_year_income", - "label": "Previous-Year Income", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_previous_year_income", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 1589, - "description": "Classify SSN card type and immigration-related CPS inputs from ASEC conditions.", - "id": "add_ssn_card_type", - "label": "Add SSN Card Type", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_ssn_card_type", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 2488, - "description": "Impute tip income and household asset inputs from SIPP donor data.", - "id": "add_tips", - "label": "Tips And Asset Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "legacy", - "target_symbol": "add_tips", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 2663, - "description": "Impute hourly wage, hourly-pay status, and union coverage from CPS ORG donors.", - "id": "add_org_inputs", - "label": "ORG Labor-Market Inputs", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "current", - "target_symbol": "add_org_labor_market_inputs", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_org.py" - ] - }, - { - "decorator_line": 2779, - "description": "Impute auto loan balance, auto loan interest, and net worth from SCF donor data.", - "id": "add_auto_loan", - "label": "Auto Loan And Net Worth Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "moving", - "status": "legacy", - "target_symbol": "add_auto_loan_interest_and_net_worth", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "decorator_line": 306, - "description": "Subsample CPS arrays for released CPS vintages while full variants skip this step.", - "id": "downsample", - "label": "Downsample CPS", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/cps/cps.py", - "stability": "stable", - "status": "current", - "target_symbol": "downsample", - "validation_commands": [ - "uv run pytest validation/stage_1/test_cps.py" - ] - }, - { - "artifacts_in": [ - "extended_cps_2024" - ], - "artifacts_out": [ - "aca_2025_takeup" - ], - "decorator_line": 404, - "description": "Adds synthetic 2025 ACA take-up assignments until calibrated person-level APTC enrollment reaches the target.", - "id": "aca_2025_override", - "label": "ACA 2025 Take-Up Override", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/enhanced_cps.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "create_aca_2025_takeup_override" - }, - { - "artifacts_in": [ - "loss_matrix", - "calibration_targets" - ], - "artifacts_out": [ - "enhanced_cps_weights" - ], - "decorator_line": 487, - "description": "Fits enhanced CPS weights against calibration targets with the hard-concrete loss machinery.", - "id": "reweight", - "label": "Enhanced CPS Reweighting", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/enhanced_cps.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "reweight" - }, - { - "artifacts_in": [ - "qrf_pass2", - "record_double" - ], - "artifacts_out": [ - "clone_feature_splice" - ], - "decorator_line": 382, - "description": "Replaces clone-half CPS feature variables with donor-matched predictions so doubled records retain plausible demographics and occupation labels.", - "id": "clone_features", - "label": "Splice Clone Features", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "_splice_clone_feature_predictions" - }, - { - "artifacts_in": [ - "record_double", - "preprocess_cps" - ], - "artifacts_out": [ - "cps_only_predictions" - ], - "decorator_line": 422, - "description": "Runs the second-stage CPS-only QRF imputation for PUF clone records inside the extended CPS build.", - "id": "cps_only", - "label": "Impute CPS-Only Variables", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "_impute_cps_only_variables" - }, - { - "artifacts_in": [ - "cps_only_predictions" - ], - "artifacts_out": [ - "extended_cps_stage2" - ], - "decorator_line": 700, - "description": "Writes second-stage CPS-only QRF predictions back into the PUF clone half of the extended CPS record set.", - "id": "qrf_pass2", - "label": "Splice CPS-Only Predictions", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "_splice_cps_only_predictions" - }, - { - "artifacts_in": [ - "extended_cps_stage2" - ], - "artifacts_out": [ - "formula_pruned_extended_cps" - ], - "decorator_line": 1179, - "description": "Removes variables computed by policyengine-us formulas, while preserving selected imputed inputs under canonical leaf variable names.", - "id": "formula_drop", - "label": "Drop Formula Variables", - "node_type": "process", - "pathways": [ - "data_build" - ], - "pydoc": true, - "source_file": "policyengine_us_data/datasets/cps/extended_cps.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "_drop_formula_variables" - }, - { - "decorator_line": 301, - "description": "Simulate Section 199A W-2 wage and UBIA guardrail quantities from PUF income.", - "id": "simulate_qbi", - "label": "QBI Simulation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "status": "current", - "target_symbol": "simulate_w2_and_ubia_from_puf", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 405, - "description": "Impute pre-tax retirement contributions onto PUF tax units from CPS donors.", - "id": "impute_puf_pension", - "label": "Impute PUF Pension Contributions", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "status": "current", - "target_symbol": "impute_pension_contributions_to_puf", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 461, - "description": "Impute missing PUF demographics from demographic donor records.", - "id": "impute_puf_demographics", - "label": "Impute PUF Demographics", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "status": "current", - "target_symbol": "impute_missing_demographics", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 620, - "description": "Rename IRS variables and derive PolicyEngine-ready PUF tax inputs.", - "id": "preprocess_puf", - "label": "Preprocess PUF", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/datasets/puf/puf.py", - "stability": "moving", - "status": "current", - "target_symbol": "preprocess_puf", - "validation_commands": [ - "uv run pytest tests/unit/datasets/test_irs_puf.py" - ] - }, - { - "decorator_line": 45, - "description": "Impute SCF-backed tax-unit mortgage balance hints for structural mortgage conversion.", - "id": "mortgage_hints", - "label": "Mortgage Balance Hint Imputation", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/utils/mortgage_interest.py", - "stability": "moving", - "status": "current", - "target_symbol": "impute_tax_unit_mortgage_balance_hints", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_mortgage_interest.py" - ] - }, - { - "decorator_line": 126, - "description": "Convert deductible mortgage interest into structural mortgage balances, interest, and origination-year inputs.", - "id": "mortgage_convert", - "label": "Structural Mortgage Conversion", - "node_type": "library", - "pathways": [ - "data_build" - ], - "source_file": "policyengine_us_data/utils/mortgage_interest.py", - "stability": "moving", - "status": "current", - "target_symbol": "convert_mortgage_interest_to_structural_inputs", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_mortgage_interest.py" - ] - } - ] -} \ No newline at end of file diff --git a/backend/data/pipeline/stages/data_build.md b/backend/data/pipeline/stages/data_build.md deleted file mode 100644 index 530de36..0000000 --- a/backend/data/pipeline/stages/data_build.md +++ /dev/null @@ -1,95 +0,0 @@ -# Data build pathway - -## Purpose - -The `data_build` pathway constructs the **enhanced CPS microdata file** that everything downstream of calibration consumes. It starts from raw Census CPS ASEC plus IRS PUF tax records, layers in donor imputations from ACS/SIPP/SCF/ORG, doubles records to splice plausible high-AGI tax filers into the survey, assigns synthetic geography, and stratifies the result down to a calibration-sized sample. It is intentionally a *separate* pathway from `calibration_package` (which packages and ships the artifacts the api/app consume) and from `local_h5` (which is the developer escape hatch for editing an h5 in place without re-running the build). Re-running `data_build` is expensive (QRF training, microsimulation passes, h5 IO); calibration runs typically reuse a cached `extended_cps_2024.h5` or `stratified_extended_cps_2024.h5`. - -## Inputs and outputs - -**Upstream sources** -- **CPS ASEC** raw person/household frames (e.g. RESNSS1/RESNSS2 SS-source codes, SS_VAL, SEMP_VAL, H_TENURE, A_AGE) — primary survey backbone. -- **IRS PUF** tax records — donor for high-AGI tax variables (`PUF_REPORTED_CALCULATED_TAX_OUTPUT_VARIABLES`). -- **ACS** (`ACS_2022`) — donor for rent and real-estate taxes with state-FIPS predictor. -- **SIPP** — donor for tip income, bank/stock/bond assets, vehicles. -- **SCF** (`SCF_2022`) — donor for net worth, auto loans, mortgage balance hints. -- **CPS-ORG** — donor for hourly wage, paid-hourly flag, union coverage. -- **Census block × CD distributions** (`block_cd_distributions.csv.gz`) — population-weighted lookup for geography assignment. - -**Outputs** -- `extended_cps_2024.h5` — full doubled (CPS half + PUF-cloned half), QRF-imputed dataset with geography assigned. -- `stratified_extended_cps_2024.h5` — calibration-sized (~30k household) stratified sample preserving the high-AGI tail. -- `source_imputed_stratified_extended_cps_2024.h5` — stratified file with ACS/SIPP/ORG/SCF imputations re-run (this is what calibration loss matrices are built against). - -## Key sub-stages (ordered) - -1. **Raw CPS construction** (`datasets/cps/cps.py`) — `add_id_variables`, `add_personal_variables`, `add_personal_income_variables`, `add_household_variables`, `add_spm_variables`, `add_ssn_card_type`, `add_previous_year_income`. Builds person/tax-unit/SPM-unit ID skeleton and populates ASEC-derived demographics, SS sub-component classification (RESNSS codes), income, prior-year income link, and immigration status. `downsample` optionally subsamples for released vintages. - -2. **PUF preprocessing + clone preparation** (`datasets/puf/puf.py`) — `preprocess_puf` renames IRS variables, `impute_puf_demographics` fills missing PUF demographics from donor records, `impute_puf_pension` (CPS donor) and `simulate_qbi` (Section 199A W-2 wage and UBIA synthesis from QBI components) populate the variables PUF doesn't carry natively. - -3. **Doubled-record build / "extended CPS"** (`calibration/puf_impute.py`, `datasets/cps/extended_cps.py`) — `record_double`/`puf_clone_dataset` concatenates each CPS household with a clone whose tax variables come from PUF via QRF (`puf_qrf_pass`). Clone half gets `household_weight=0` so it only contributes once calibration picks it up. `weeks_impute` and `retire_impute` fill clone-half labor/retirement holes; `ss_reconcile` scales Social Security sub-components so they sum to the imputed total. A second QRF pass (`cps_only` → `qrf_pass2` → `clone_features`) replaces naive donor copies of CPS-only variables on the clone half with predictions consistent with the clone's imputed PUF income. `formula_drop` strips variables policyengine-us will compute via formula so they don't poison reforms. - -4. **Mortgage structural conversion** (`utils/mortgage_interest.py`) — `mortgage_hints` imputes SCF-backed first/second-home balance hints, then `mortgage_convert` rewrites formula-level `deductible_mortgage_interest` and `interest_deduction` into structural mortgage balances + origination years + person-level home/investment interest. Required so reforms to the MID don't no-op against pre-imputed deductibles. - -5. **ACA take-up override** (`datasets/cps/enhanced_cps.py`) — `aca_2025_override` adds synthetic 2025 marketplace take-up draws until calibrated person-level APTC enrollment hits national (or per-state) targets. - -6. **Stratification, source re-imputation, geography assignment** (`calibration/create_stratified_cps.py`, `calibration/source_impute.py`, `calibration/clone_and_assign.py`) — `create_stratified` subsamples to ~30k households with per-bracket caps on the high-AGI tail (`HIGH_AGI_BRACKETS`, capping the >$10M PUF pile-up at 300 and the $1M-$2M middle-high band at 400) plus optional bottom-quartile oversample. `source_impute` re-runs ACS/SIPP/SCF/ORG QRFs on the stratified subset (ACS+ORG use state_fips as a predictor; SIPP+SCF don't). `geo_assign` draws population-weighted census blocks for every clone from `block_cd_distributions.csv.gz`, optionally reweighting blocks within a CD by AGI target shares; state/CD/county/tract GEOIDs are derived from the block GEOID. - -## Common failure modes - -- **`extended_cps_2024.h5` missing or stale**: `create_stratified_cps_dataset` and `source_impute` both default to `STORAGE_FOLDER/extended_cps_2024.h5`. If a partial build wrote a corrupt file, `add_rent`'s stale-key warning (`cps.py:362-379`) is usually the first symptom; delete the h5 and rerun. Stage assignment for the calibration matrix will silently use stale variable values otherwise. -- **`block_cd_distributions.csv.gz` not present**: `load_global_block_distribution` (`clone_and_assign.py:48`) raises `FileNotFoundError` with a pointer to `make_block_cd_distributions.py`. The lru_cache means the first failure poisons subsequent calls in the same process. -- **QRF predictor missing in CPS at imputation time**: any of the `_impute_*` helpers in `puf_impute.py` and `source_impute.py` instantiate a fresh `Microsimulation(dataset=dataset_path)` to compute predictors. If the CPS h5 lacks a predictor variable (renamed upstream in policyengine-us or dropped by `formula_drop`), calculate raises and the surrounding `try/except` (e.g. `_impute_weeks_unemployed` at `puf_impute.py:662-667`) may silently return zeros. Check logs for "weeks_unemployed not in CPS" / "returning zeros". -- **SS sub-components don't reconcile**: `reconcile_ss_subcomponents` (`puf_impute.py:384`) only fixes the PUF half (indices `n_cps:`). If a CPS-half sub-component classification (`cps.py:1190+`) misroutes due to ASEC code changes, totals on the CPS half can drift from `social_security` and the calibration target on retirement vs. disability rolls will be off. -- **AGI-tail bracket starvation in stratified output**: if PUF templates with `household_weight=0` are missing or mislabeled, `create_stratified_cps_dataset` (`create_stratified_cps.py:156-180`) will draw fewer than `cap` records and the high-AGI calibration targets become noisy. The function prints per-bracket counts — check them whenever calibration tail metrics regress. -- **Geography QA**: at-large CD normalization (CD 00 / DC 98 → 01) happens in `load_global_block_distribution`. If a new release changes Census's at-large encoding, district-level targets will misalign without an obvious error. - -## Why this exists / design notes - -The pathway exists to solve four entangled problems no single source can handle: -1. **CPS top-codes income** around $1M, but tax policy lives in the upper tail. Cloning each CPS household and imputing PUF tax variables onto the clone (with `household_weight=0`) lets the calibration optimizer assign positive weight to high-AGI synthetic households without distorting low-income statistics. -2. **PUF lacks demographics and transfers**; CPS lacks tax detail. A two-pass QRF (PUF → clone, then CPS-only → clone) propagates each donor's strengths while keeping within-clone consistency between income and CPS-only variables (`extended_cps.py:439+`). -3. **Several variables policyengine-us computes via formula** (e.g. mortgage interest deductions, ACA take-up) need structural inputs, not pre-imputed outputs, or reforms become no-ops. The mortgage conversion and ACA override stages exist purely to invert formula-level imputations into structural inputs. -4. **Calibration runs on a stratified subset for tractability**, but naive subsampling would lose the high-AGI tail that drives most revenue-cost reforms. `HIGH_AGI_BRACKETS` per-bracket caps keep the tail bounded while preferring weighted CPS records over `household_weight=0` PUF templates within each cap. - -A subtle point for reviewers: the clone-half `household_weight=0` is load-bearing. Every transformation in this stage assumes the first `n_cps` indices are the original CPS records and indices `n_cps:` are the PUF clones. Search for `n_cps:` or `n_half:` in `puf_impute.py` and `extended_cps.py` — most splicing logic is hard-coded around that contract. - -## Source map - -| Node id | Source file | Role | -|---|---|---| -| add_id_variables | `datasets/cps/cps.py:918` | Build person/household/tax-unit/SPM/family/marital ID skeleton | -| add_personal_variables | `datasets/cps/cps.py:982` | Populate demographics and occupation-derived inputs | -| add_personal_income_variables | `datasets/cps/cps.py:1126` | Populate income, transfers, retirement, QBI inputs; classify SS by RESNSS codes | -| add_spm_variables | `datasets/cps/cps.py:1402` | Populate SPM-unit poverty variables | -| add_household_variables | `datasets/cps/cps.py:1441` | Populate household geography (state, county, NYC flag) | -| add_previous_year_income | `datasets/cps/cps.py:1483` | Link adjacent CPS years for prior-year income | -| add_ssn_card_type | `datasets/cps/cps.py:1589` | Classify SSN card type / immigration status from ASEC | -| add_rent | `datasets/cps/cps.py:339` | Legacy ACS-donor rent and real-estate-tax imputation | -| add_takeup | `datasets/cps/cps.py:463` | Stochastic benefit takeup with reported-anchor alignment | -| add_tips | `datasets/cps/cps.py:2488` | Legacy SIPP-donor tip and asset imputation | -| add_org_inputs | `datasets/cps/cps.py:2663` | ORG-donor hourly wage, hourly-pay flag, union coverage | -| add_auto_loan | `datasets/cps/cps.py:2779` | Legacy SCF-donor auto loan and net worth imputation | -| downsample | `datasets/cps/cps.py:306` | Subsample CPS arrays for released vintages | -| preprocess_puf | `datasets/puf/puf.py:620` | Rename IRS variables, derive PE-ready PUF inputs | -| impute_puf_demographics | `datasets/puf/puf.py:461` | Fill missing PUF demographics from donor records | -| impute_puf_pension | `datasets/puf/puf.py:405` | Impute pre-tax retirement contributions onto PUF | -| simulate_qbi | `datasets/puf/puf.py:301` | Synthesize Section 199A W-2 wages and UBIA | -| record_double | `calibration/puf_impute.py:436` | Double CPS records; clone-half gets PUF tax imputations | -| puf_qrf_pass | `calibration/puf_impute.py:875` | QRF imputation from PUF tax variables onto clones | -| weeks_impute | `calibration/puf_impute.py:622` | Impute weeks unemployed for clone half | -| retire_impute | `calibration/puf_impute.py:730` | Impute retirement contribution inputs for clone half | -| ss_reconcile | `calibration/puf_impute.py:369` | Scale SS sub-components to reconcile to total SS (PUF half) | -| cps_only | `datasets/cps/extended_cps.py:422` | Second-stage CPS-only QRF for PUF clone records | -| qrf_pass2 | `datasets/cps/extended_cps.py:700` | Splice CPS-only QRF predictions into clone half | -| clone_features | `datasets/cps/extended_cps.py:382` | Replace clone-half feature variables with donor-matched predictions | -| formula_drop | `datasets/cps/extended_cps.py:1179` | Drop variables that policyengine-us computes via formula | -| mortgage_hints | `utils/mortgage_interest.py:45` | SCF-backed tax-unit mortgage balance hint imputation | -| mortgage_convert | `utils/mortgage_interest.py:126` | Convert deductible MID into structural mortgage inputs | -| aca_2025_override | `datasets/cps/enhanced_cps.py:404` | Synthetic 2025 ACA take-up to match APTC targets | -| reweight | `datasets/cps/enhanced_cps.py:487` | Fit enhanced CPS weights against calibration targets (hard-concrete loss) | -| create_stratified | `calibration/create_stratified_cps.py:68` | Stratified sample with high-AGI tail caps | -| source_impute | `calibration/source_impute.py:170` | Re-impute ACS/SIPP/ORG/SCF on stratified file | -| acs_qrf | `calibration/source_impute.py:319` | ACS QRF: rent + real-estate taxes (state predictor) | -| sipp_qrf | `calibration/source_impute.py:420` | SIPP QRF: tips, liquid assets, vehicles | -| scf_qrf | `calibration/source_impute.py:804` | SCF QRF: net worth, auto loans, balance-sheet components | -| geo_assign | `calibration/clone_and_assign.py:147` | Assign population-weighted blocks → CD/county/state to clones | diff --git a/backend/data/pipeline/stages/local_h5.input.json b/backend/data/pipeline/stages/local_h5.input.json deleted file mode 100644 index ee7110f..0000000 --- a/backend/data/pipeline/stages/local_h5.input.json +++ /dev/null @@ -1,984 +0,0 @@ -{ - "stage_id": "local_h5", - "label": "local_h5", - "nodes": [ - { - "decorator_line": 37, - "description": "Build typed local H5 requests from US states, districts, and supported city rules.", - "id": "local_h5_area_catalog", - "label": "USAreaCatalog", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/area_catalog.py", - "stability": "moving", - "status": "current", - "target_symbol": "USAreaCatalog", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_area_catalog.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db" - ], - "artifacts_out": [ - "worker_bootstrap.json", - "entity_graph.npz" - ], - "decorator_line": 52, - "description": "Persisted deterministic worker setup facts for one local H5 scope.", - "id": "local_h5_worker_bootstrap_bundle", - "label": "WorkerBootstrapBundle", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/bootstrap.py", - "stability": "moving", - "status": "current", - "target_symbol": "WorkerBootstrapBundle", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_bootstrap.py" - ] - }, - { - "artifacts_out": [ - "bootstrap/{scope}/worker_bootstrap.json", - "bootstrap/{scope}/entity_graph.npz" - ], - "decorator_line": 198, - "description": "Filesystem path adapter for run-scoped local H5 bootstrap artifacts.", - "id": "local_h5_worker_bootstrap_store", - "label": "WorkerBootstrapStore", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/bootstrap.py", - "stability": "moving", - "status": "current", - "target_symbol": "WorkerBootstrapStore", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_bootstrap.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db" - ], - "artifacts_out": [ - "bootstrap/{scope}/worker_bootstrap.json", - "bootstrap/{scope}/entity_graph.npz" - ], - "decorator_line": 259, - "description": "Materialize deterministic local H5 worker bootstrap artifacts.", - "id": "local_h5_worker_bootstrap_builder", - "label": "WorkerBootstrapBuilder", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/bootstrap.py", - "stability": "moving", - "status": "current", - "target_symbol": "WorkerBootstrapBuilder", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_bootstrap.py" - ] - }, - { - "decorator_line": 71, - "description": "In-memory local H5 payload and diagnostics for one area.", - "id": "local_h5_build_result", - "label": "LocalAreaBuildResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/builder.py", - "stability": "moving", - "status": "current", - "target_symbol": "LocalAreaBuildResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_builder.py" - ] - }, - { - "decorator_line": 139, - "description": "Build the in-memory payload for one local-area or national H5 output.", - "id": "local_h5_dataset_builder", - "label": "LocalAreaDatasetBuilder", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/builder.py", - "stability": "moving", - "status": "current", - "target_symbol": "LocalAreaDatasetBuilder", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_builder.py" - ] - }, - { - "decorator_line": 31, - "description": "Input artifact and run metadata bundle for one local H5 publish scope.", - "id": "local_h5_publishing_input_bundle", - "label": "PublishingInputBundle", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "moving", - "status": "current", - "target_symbol": "PublishingInputBundle" - }, - { - "decorator_line": 78, - "description": "Stable content identity for one local H5 publication input artifact.", - "id": "local_h5_artifact_identity", - "label": "ArtifactIdentity", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "stable", - "status": "current", - "target_symbol": "ArtifactIdentity", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "decorator_line": 113, - "description": "Provenance and resumability material for one local H5 publish scope.", - "id": "local_h5_traceability_bundle", - "label": "TraceabilityBundle", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "moving", - "status": "current", - "target_symbol": "TraceabilityBundle", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "decorator_line": 188, - "description": "Build traceability bundles and deterministic scope fingerprints for local H5 publication.", - "id": "local_h5_traceability", - "label": "FingerprintingService", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/fingerprinting.py", - "stability": "moving", - "status": "current", - "target_symbol": "FingerprintingService", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "decorator_line": 40, - "description": "Resolved physical source used to recover exact calibration geography.", - "id": "local_h5_resolved_geography_source", - "label": "ResolvedGeographySource", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/geography_loader.py", - "stability": "stable", - "status": "current", - "target_symbol": "ResolvedGeographySource", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_geography_loader.py" - ] - }, - { - "decorator_line": 77, - "description": "Resolve and load saved, package-backed, or legacy calibration geography artifacts.", - "id": "calibration_geography_loader", - "label": "CalibrationGeographyLoader", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/geography_loader.py", - "stability": "moving", - "status": "current", - "target_symbol": "CalibrationGeographyLoader", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_geography_loader.py" - ] - }, - { - "decorator_line": 29, - "description": "Assign weighted area work items to worker batches using longest-processing-time scheduling.", - "id": "local_h5_partition", - "label": "Partition Local H5 Work", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/partitioning.py", - "stability": "stable", - "status": "current", - "target_symbol": "partition_weighted_work_items", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_partitioning.py" - ] - }, - { - "decorator_line": 43, - "description": "Validated in-memory period-grouped payload for one local H5 output.", - "id": "local_h5_payload", - "label": "H5Payload", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/payload.py", - "stability": "moving", - "status": "current", - "target_symbol": "H5Payload", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_payload.py" - ] - }, - { - "decorator_line": 122, - "description": "Context passed to payload postprocessors during one H5 build.", - "id": "local_h5_payload_build_context", - "label": "PayloadBuildContext", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/payload.py", - "stability": "moving", - "status": "current", - "target_symbol": "PayloadBuildContext", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_payload.py" - ] - }, - { - "decorator_line": 20, - "description": "Reindex selected household, person, and subentity rows for local H5 outputs.", - "id": "local_h5_entity_reindexer", - "label": "EntityReindexer", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/reindexing.py", - "stability": "moving", - "status": "current", - "target_symbol": "EntityReindexer", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_reindexing.py" - ] - }, - { - "decorator_line": 111, - "description": "Output entity IDs and source row indices after local H5 clone selection.", - "id": "local_h5_reindexed_entities", - "label": "ReindexedEntities", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/reindexing.py", - "stability": "moving", - "status": "current", - "target_symbol": "ReindexedEntities", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_reindexing.py" - ] - }, - { - "decorator_line": 61, - "description": "Typed geography predicate for local H5 output selection.", - "id": "local_h5_area_filter", - "label": "AreaFilter", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/requests.py", - "stability": "moving", - "status": "current", - "target_symbol": "AreaFilter", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_requests.py" - ] - }, - { - "decorator_line": 137, - "description": "Typed request contract for one national, state, district, city, or custom H5 output.", - "id": "local_h5_area_request", - "label": "AreaBuildRequest", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/requests.py", - "stability": "moving", - "status": "current", - "target_symbol": "AreaBuildRequest", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_requests.py" - ] - }, - { - "decorator_line": 19, - "description": "Select active clone-household rows for one local H5 output.", - "id": "local_h5_area_selector", - "label": "AreaSelector", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/selection.py", - "stability": "moving", - "status": "current", - "target_symbol": "AreaSelector", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_selection.py" - ] - }, - { - "decorator_line": 101, - "description": "Selected clone-household rows for one local H5 output.", - "id": "local_h5_clone_selection", - "label": "CloneSelection", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/selection.py", - "stability": "moving", - "status": "current", - "target_symbol": "CloneSelection", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_selection.py" - ] - }, - { - "decorator_line": 63, - "description": "Explicit source-dataset entity spine for local H5 worker setup.", - "id": "local_h5_entity_graph", - "label": "EntityGraph", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "status": "current", - "target_symbol": "EntityGraph", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_source_dataset.py" - ] - }, - { - "decorator_line": 370, - "description": "Lazy source variable access wrapper for local H5 source snapshots.", - "id": "local_h5_microsimulation_variable_provider", - "label": "MicrosimulationVariableProvider", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "status": "current", - "target_symbol": "MicrosimulationVariableProvider", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_source_dataset.py" - ] - }, - { - "decorator_line": 469, - "description": "In-memory source H5 dataset contract for local H5 worker setup.", - "id": "local_h5_source_dataset_snapshot", - "label": "SourceDatasetSnapshot", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "status": "current", - "target_symbol": "SourceDatasetSnapshot", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_source_dataset.py" - ] - }, - { - "decorator_line": 546, - "description": "PolicyEngine H5 dataset adapter for local H5 source snapshots.", - "id": "local_h5_policyengine_dataset_reader", - "label": "PolicyEngineDatasetReader", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/source_dataset.py", - "stability": "moving", - "status": "current", - "target_symbol": "PolicyEngineDatasetReader", - "validation_commands": [ - "uv run pytest tests/integration/build_outputs/h5_worker_runtime/test_worker_script_tiny_fixture.py" - ] - }, - { - "decorator_line": 54, - "description": "US entity ID and household-weight local H5 payload data.", - "id": "local_h5_us_entity_postprocessor_result", - "label": "USEntityPostProcessorResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "status": "current", - "target_symbol": "USEntityPostProcessorResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 80, - "description": "US geography local H5 payload data and block-derived geography.", - "id": "local_h5_us_geography_postprocessor_result", - "label": "USGeographyPostProcessorResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "status": "current", - "target_symbol": "USGeographyPostProcessorResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 107, - "description": "US take-up local H5 payload data and generated take-up variables.", - "id": "local_h5_us_takeup_postprocessor_result", - "label": "USTakeupPostProcessorResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "status": "current", - "target_symbol": "USTakeupPostProcessorResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 134, - "description": "Apply US entity ID and household-weight fields to local H5 payloads.", - "id": "local_h5_us_entity_postprocessor", - "label": "USEntityPostProcessor", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "status": "current", - "target_symbol": "USEntityPostProcessor", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 187, - "description": "Apply US geography fields to local H5 payloads.", - "id": "local_h5_us_geography_postprocessor", - "label": "USGeographyPostProcessor", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "status": "current", - "target_symbol": "USGeographyPostProcessor", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 292, - "description": "Apply US take-up fields to local H5 payloads.", - "id": "local_h5_us_takeup_postprocessor", - "label": "USTakeupPostProcessor", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/us_augmentations.py", - "stability": "moving", - "status": "current", - "target_symbol": "USTakeupPostProcessor", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_us_augmentations.py" - ] - }, - { - "decorator_line": 23, - "description": "Worker-scoped local H5 validation policy contract.", - "id": "local_h5_validation_policy", - "label": "ValidationPolicy", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/validation.py", - "stability": "moving", - "status": "current", - "target_symbol": "ValidationPolicy", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_validation.py" - ] - }, - { - "decorator_line": 41, - "description": "Prepared per-worker local H5 validation target context.", - "id": "local_h5_validation_context", - "label": "ValidationContext", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/validation.py", - "stability": "moving", - "status": "current", - "target_symbol": "ValidationContext", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_validation.py" - ] - }, - { - "artifacts_in": [ - "policy_data.db", - "target_config.yaml", - "target_config_full.yaml" - ], - "decorator_line": 115, - "description": "Prepare local H5 validation targets once per worker session.", - "id": "local_h5_area_validation_service", - "label": "AreaValidationService", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/validation.py", - "stability": "moving", - "status": "current", - "target_symbol": "AreaValidationService", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_validation.py" - ] - }, - { - "decorator_line": 38, - "description": "Clone selected source variables into period-grouped local H5 payloads.", - "id": "local_h5_variable_cloner", - "label": "VariableCloner", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/variables.py", - "stability": "moving", - "status": "current", - "target_symbol": "VariableCloner", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_variables.py" - ] - }, - { - "decorator_line": 126, - "description": "Period-grouped cloned source variables for one local H5 output.", - "id": "local_h5_variable_clone_payload", - "label": "VariableClonePayload", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/variables.py", - "stability": "moving", - "status": "current", - "target_symbol": "VariableClonePayload", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_variables.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "national_calibration_weights.npy" - ], - "decorator_line": 20, - "description": "Explicit shape contract for flat clone-level calibration weights.", - "id": "clone_weight_matrix", - "label": "CloneWeightMatrix", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/weights.py", - "stability": "moving", - "status": "current", - "target_symbol": "CloneWeightMatrix", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_weights.py" - ] - }, - { - "decorator_line": 50, - "description": "Normalized worker-execution input payload for local H5 builds.", - "id": "local_h5_worker_calibration_inputs", - "label": "WorkerCalibrationInputs", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/worker_inputs.py", - "stability": "moving", - "status": "current", - "target_symbol": "WorkerCalibrationInputs", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_worker_inputs.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db", - "worker_bootstrap.json" - ], - "decorator_line": 34, - "description": "Worker-scoped local H5 setup state reused across queued requests.", - "id": "local_h5_worker_session", - "label": "WorkerSession", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/worker_session.py", - "stability": "moving", - "status": "current", - "target_symbol": "WorkerSession", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_worker_session.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps.h5", - "geography_assignment.npz", - "policy_data.db", - "bootstrap/{scope}/worker_bootstrap.json", - "bootstrap/{scope}/entity_graph.npz" - ], - "decorator_line": 69, - "description": "Load local H5 source, weights, geography, and validation context once per worker.", - "id": "local_h5_worker_session_factory", - "label": "WorkerSessionFactory", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/worker_session.py", - "stability": "moving", - "status": "current", - "target_symbol": "WorkerSessionFactory", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_worker_session.py", - "uv run pytest tests/integration/build_outputs/h5_worker_runtime/test_worker_script_tiny_fixture.py" - ] - }, - { - "decorator_line": 18, - "description": "Post-write verification summary for one local H5 file.", - "id": "local_h5_write_result", - "label": "H5WriteResult", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/writer.py", - "stability": "moving", - "status": "current", - "target_symbol": "H5WriteResult", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_writer.py" - ] - }, - { - "decorator_line": 40, - "description": "Write one period-grouped local H5 payload to disk.", - "id": "local_h5_writer", - "label": "H5Writer", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/build_outputs/writer.py", - "stability": "moving", - "status": "current", - "target_symbol": "H5Writer", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_writer.py" - ] - }, - { - "decorator_line": 540, - "description": "Derive state, county, tract, PUMA, place, district, and other geography from block GEOIDs.", - "id": "geo_derive", - "label": "Derive Geography From Blocks", - "node_type": "library", - "pathways": [ - "local_h5", - "calibration_package" - ], - "source_file": "policyengine_us_data/calibration/block_assignment.py", - "stability": "moving", - "status": "current", - "target_symbol": "derive_geography_from_blocks", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_clone_and_assign.py" - ] - }, - { - "artifacts_in": [ - "local_area_build/**/*.h5" - ], - "decorator_line": 96, - "description": "Upload locally built H5 files into Hugging Face staging paths.", - "id": "local_stage_upload", - "label": "Stage Local H5 Files", - "node_type": "entrypoint", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/promote_local_h5s.py", - "stability": "moving", - "status": "current", - "target_symbol": "stage", - "validation_commands": [ - "uv run pytest tests/unit/test_promote_local_h5s.py" - ] - }, - { - "artifacts_out": [ - "production H5 release", - "release manifest" - ], - "decorator_line": 116, - "description": "Promote staged H5 files to production storage and publish release manifests.", - "id": "atomic_promote", - "label": "Atomic Promote Local H5 Files", - "node_type": "entrypoint", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/promote_local_h5s.py", - "stability": "moving", - "status": "current", - "target_symbol": "promote", - "validation_commands": [ - "uv run pytest tests/unit/test_promote_local_h5s.py" - ] - }, - { - "api_refs": [ - "policyengine_us_data.build_outputs.fingerprinting.FingerprintingService" - ], - "decorator_line": 60, - "description": "Compute a scope fingerprint for local H5 checkpoint and resume decisions.", - "id": "local_h5_input_fingerprint", - "label": "Compute Local H5 Input Fingerprint", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "status": "legacy", - "target_symbol": "compute_input_fingerprint", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_fingerprinting.py" - ] - }, - { - "api_refs": [ - "policyengine_us_data.build_outputs.geography_loader.CalibrationGeographyLoader" - ], - "artifacts_in": [ - "calibration_weights.npy", - "geography_assignment.npz", - "stacked_blocks.npy" - ], - "decorator_line": 114, - "description": "Resolve exact geography from saved bundles, package metadata, or legacy block artifacts.", - "id": "load_calibration_geography", - "label": "Load Calibration Geography", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "status": "legacy", - "target_symbol": "load_calibration_geography", - "validation_commands": [ - "uv run pytest tests/unit/build_outputs/test_geography_loader.py" - ] - }, - { - "artifacts_in": [ - "calibration_weights.npy", - "source_imputed_stratified_extended_cps*.h5" - ], - "artifacts_out": [ - "states/*.h5", - "districts/*.h5", - "cities/*.h5", - "US.h5" - ], - "decorator_line": 306, - "description": "Expand calibrated clone weights into local-area H5 datasets with geography and takeup updates.", - "details": "This is the main bundled H5 construction routine and remains a critical transitional waypoint.", - "id": "build_h5", - "label": "Build Local Area H5", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "build_h5", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_publish_local_area.py", - "uv run pytest tests/integration/test_tiny_h5_pipeline.py" - ] - }, - { - "artifacts_out": [ - "states/*.h5" - ], - "decorator_line": 441, - "description": "Build state-level H5 files from calibrated weights and exact geography.", - "id": "build_states", - "label": "Build State H5 Files", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "status": "current", - "target_symbol": "build_states" - }, - { - "artifacts_out": [ - "districts/*.h5" - ], - "decorator_line": 527, - "description": "Build congressional-district H5 files from calibrated weights and exact geography.", - "id": "build_districts", - "label": "Build District H5 Files", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "status": "current", - "target_symbol": "build_districts" - }, - { - "artifacts_out": [ - "cities/*.h5" - ], - "decorator_line": 614, - "description": "Build supported city H5 files with county probability filtering.", - "id": "build_cities", - "label": "Build City H5 Files", - "node_type": "library", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/publish_local_area.py", - "stability": "moving", - "status": "current", - "target_symbol": "build_cities" - }, - { - "decorator_line": 312, - "description": "Check calibrated H5 structure, weights, IDs, mappings, takeup, and aggregate sanity.", - "id": "sanity_checks", - "label": "Run H5 Sanity Checks", - "node_type": "validation", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/sanity_checks.py", - "stability": "moving", - "status": "current", - "target_symbol": "run_sanity_checks", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_validate_staging.py" - ] - }, - { - "decorator_line": 302, - "description": "Run microsimulation target comparisons for one staged area.", - "id": "target_validation", - "label": "Validate Area Against Targets", - "node_type": "validation", - "pathways": [ - "local_h5" - ], - "source_file": "policyengine_us_data/calibration/validate_staging.py", - "stability": "moving", - "status": "current", - "target_symbol": "validate_area", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_validate_staging.py" - ] - } - ] -} \ No newline at end of file diff --git a/backend/data/pipeline/stages/local_h5.md b/backend/data/pipeline/stages/local_h5.md deleted file mode 100644 index 1a361c7..0000000 --- a/backend/data/pipeline/stages/local_h5.md +++ /dev/null @@ -1,115 +0,0 @@ -# Local-area H5 build pathway - -## Purpose - -This stage materializes the per-area microsimulation datasets that PolicyEngine ships to production: one H5 file per US state (`states/CA.h5`), per congressional district (`districts/TX-15.h5`), per supported city (`cities/NYC.h5`), plus a national `US.h5`. It exists as a separate pathway from `calibration_package` because calibration produces only a single flat clone-weight vector aligned to a cloned source frame; this stage *expands* those weights into many self-contained, geography-filtered H5 files that downstream consumers (the API, app, district analyses) can load independently. It is also intentionally split from `data_build`: `data_build` produces the *source* CPS/PUF/imputed snapshot once, while `local_h5` runs ~480 times per release (50 states + ~436 districts + cities + national), is embarrassingly parallel, must be checkpointable across worker crashes, and ends in a multi-stage HuggingFace staging plus atomic promotion that `data_build` does not need. - -## Inputs and outputs - -**Inputs** (from upstream stages, consumed by `WorkerSessionFactory` and `build_h5`): -- `source_imputed_stratified_extended_cps.h5` — the imputed/extended CPS source frame from `data_build`. -- `calibration_weights.npy` — flat clone-level weight vector `(n_clones_total * n_hh,)` from `calibration_package`. -- `geography_assignment.npz` — per-clone block-level geography assignment (block GEOID, CD GEOID) recovered via `CalibrationGeographyLoader`. -- `policy_data.db` + `target_config.yaml` / `target_config_full.yaml` — used by `AreaValidationService` for post-build target comparisons. -- Optional `bootstrap/{scope}/worker_bootstrap.json` + `entity_graph.npz` — persisted deterministic worker setup facts, when available. - -**Outputs**: -- `states/*.h5`, `districts/*.h5`, `cities/*.h5`, `US.h5` — geography-filtered H5 files with reindexed entity IDs, household weights, derived geography (state/county/tract/PUMA/place), and take-up draws applied. -- Release manifest + HF release tag (after `atomic_promote`). -- Per-area validation diagnostics (sanity checks + target comparisons). - -## Key sub-stages - -1. **Request enumeration** — `local_h5_area_catalog` (`USAreaCatalog`), `local_h5_area_request` (`AreaBuildRequest`), `local_h5_area_filter` (`AreaFilter`). The catalog walks the unique CD GEOIDs in the geography assignment and emits one typed `AreaBuildRequest` per state, district, supported city (NYC = county-FIPS filter), and the national output. Each request carries an `output_relative_path` and an immutable tuple of `AreaFilter`s. - -2. **Worker setup and bootstrap** — `local_h5_worker_session_factory`, `local_h5_worker_session`, `local_h5_worker_bootstrap_bundle`, `local_h5_worker_bootstrap_builder`, `local_h5_partition`. The factory loads the source H5 (via `PolicyEngineDatasetReader`), the calibration weights, the geography (preferring persisted bootstrap artifacts under `bootstrap/{scope}/`, falling back to raw loaders), and the validation context *once per worker process*. `partition_weighted_work_items` uses longest-processing-time scheduling to balance areas across workers and respects a `completed` set for resumability. - -3. **In-memory build per area** — `local_h5_dataset_builder` (`LocalAreaDatasetBuilder`), `local_h5_area_selector`, `local_h5_clone_selection`, `local_h5_entity_reindexer`, `local_h5_reindexed_entities`, `local_h5_variable_cloner`, `local_h5_payload`. For one request, `AreaSelector` zeroes out clones outside the geography filter and returns `(clone_indices, source_household_indices, weights, block_geoids)`. `EntityReindexer` produces new contiguous IDs for households/persons/tax units/spm units/families. `VariableCloner` materializes period-grouped arrays into an `H5Payload`. - -4. **US postprocessing** — `local_h5_us_entity_postprocessor`, `local_h5_us_geography_postprocessor`, `local_h5_us_takeup_postprocessor`. In order: write the new entity IDs and the calibrated `household_weight`; derive state/county/tract/PUMA/place/district from block GEOIDs (`derive_geography_from_blocks` in `calibration/block_assignment.py`); apply take-up draws for ACA/SNAP/etc. The order is enforced by `PayloadPostProcessorSpec.requires` so geography fields exist before take-up depends on them. - -5. **Write + verify** — `local_h5_writer` (`H5Writer`), `local_h5_write_result`. Writes the payload as period-grouped datasets and reads back household/person counts and weight sums for immediate verification. - -6. **Validate, stage, promote** — `sanity_checks` (`run_sanity_checks`), `target_validation` (`validate_area`), `local_stage_upload` (`stage`), `atomic_promote` (`promote`). Sanity checks open the H5 and verify weight non-negativity, ID uniqueness, person/household mapping integrity, take-up coherence, and aggregate sanity. Target validation runs a fresh microsim against the H5 and compares to the calibration target table. `stage` uploads to `staging/{run_id}/...` on HF; `promote` runs preflight (`preflight_release_manifest_publish`), moves staging to production, mirrors to GCS, publishes the release manifest, and only creates the version tag when every required area prefix is present. - -## Common failure modes - -- **"No active clones after filtering"** — raised in `AreaSelector.select` (`selection.py:67`). The geography filter for the area matched zero clones with positive weight. Usually means the calibration weight vector and `geography_assignment.npz` are misaligned (different clone counts), or a stale `state_filter` / wrong CD GEOID list. Check `CalibrationGeographyLoader` resolution first — it has saved/package/legacy fallback paths and may have loaded the wrong one. - -- **"N active clones have empty block GEOIDs"** — also in `AreaSelector.select` (`selection.py:81`). The geography assignment has unassigned blocks for clones that survived filtering. Indicates upstream block assignment is incomplete; look at `calibration/block_assignment.py` and the source of `stacked_blocks.npy`. - -- **Bootstrap fingerprint mismatch / `bootstrap_status == "fallback"`** — `WorkerSessionFactory.create` validates `expected_scope_fingerprint` against the persisted bootstrap bundle and silently falls back to raw loaders if it disagrees. Builds will succeed but much more slowly. Check `FingerprintingService` and the `PublishingInputBundle` artifact hashes; usually the source H5 or weights were rebuilt without invalidating bootstrap. - -- **Sanity-check WARN/FAIL on hourly wage or weights** — `run_sanity_checks` in `sanity_checks.py:327` checks weight non-negativity, person→household weight propagation, and hourly-wage-vs-income consistency. Failures here typically point at the take-up postprocessor or at a person→household weight broadcast mismatch in `USEntityPostProcessor` (entity ID arrays out of sync with the cloned variable arrays). - -- **Atomic promote refuses to tag** — `preflight_release_manifest_publish` reports `missing_prefixes` (e.g., `cities/` not yet built). `promote()` still copies staged files but logs a warning and leaves the release untagged. Recover by running the missing area type and re-invoking `promote`. - -## Why this exists / design notes - -The original pathway (`build_h5` in `publish_local_area.py`, still tagged `status="transitional"`) was a single monolithic function that loaded the source simulation, filtered clones, reindexed entities, cloned variables, applied US-specific augmentations, and wrote H5 — all in one call. That worked for the national output but became a bottleneck once PolicyEngine started shipping ~500 area-specific files per release: each call re-loaded the source dataset, there was no clean seam for tests, and worker crashes lost all in-flight work. - -The `build_outputs/` package is a refactor that pulls each concern into a typed seam — `AreaSelector`, `EntityReindexer`, `VariableCloner`, the three `USPostProcessor`s, `H5Writer` — coordinated by `LocalAreaDatasetBuilder`. `WorkerSession`/`WorkerSessionFactory` amortize the expensive setup (source H5 read, weight load, geography load, validation target prep) across many area builds in the same worker. `WorkerBootstrap*` persists those setup facts so a restarted worker can skip re-deriving the entity graph. `partition_weighted_work_items` ensures that a worker dying mid-shard does not stall the longest build. - -Subtle points for reviewers: -- The `status` field is meaningful: `build_h5` is `transitional`, the `publish_local_area.py` `compute_input_fingerprint` and `load_calibration_geography` are `legacy` — the new equivalents live in `build_outputs/fingerprinting.py` and `build_outputs/geography_loader.py`. -- Postprocessor ordering is enforced via `PayloadPostProcessorSpec.requires` and validated in `LocalAreaDatasetBuilder.__post_init__`. Take-up depends on geography (it filters by block), so adding a postprocessor that reads geography before geography is applied will silently get pre-geography data unless declared. -- The pathway is dual-keyed in the registry: `derive_geography_from_blocks` (`geo_derive`) is shared with `calibration_package` because both pathways need to translate blocks to higher geography levels. -- Promotion is intentionally non-atomic across files but atomic on the *tag*: partial promotes are recoverable, but a release is only "real" once the tag exists. - -## Source map - -| Node ID | Source file | Role | -|---|---|---| -| local_h5_area_catalog | build_outputs/area_catalog.py | Build typed `AreaBuildRequest`s from US geography (states, CDs, NYC) | -| local_h5_area_request | build_outputs/requests.py | Typed contract for one H5 output request | -| local_h5_area_filter | build_outputs/requests.py | Geography predicate (`field op value`) | -| local_h5_worker_bootstrap_bundle | build_outputs/bootstrap.py | Persisted deterministic worker setup facts | -| local_h5_worker_bootstrap_store | build_outputs/bootstrap.py | Run-scoped path adapter for bootstrap artifacts | -| local_h5_worker_bootstrap_builder | build_outputs/bootstrap.py | Materialize bootstrap JSON + entity-graph NPZ | -| local_h5_worker_session | build_outputs/worker_session.py | Per-worker reusable state (source, weights, geography) | -| local_h5_worker_session_factory | build_outputs/worker_session.py | Construct `WorkerSession` from bootstrap or raw loaders | -| local_h5_worker_calibration_inputs | build_outputs/worker_inputs.py | Normalized worker-input payload | -| local_h5_partition | build_outputs/partitioning.py | LPT scheduling of weighted area work across workers | -| local_h5_publishing_input_bundle | build_outputs/fingerprinting.py | Input artifact + run metadata bundle | -| local_h5_artifact_identity | build_outputs/fingerprinting.py | Content-addressed identity for one input artifact | -| local_h5_traceability_bundle | build_outputs/fingerprinting.py | Provenance + resumability material | -| local_h5_traceability | build_outputs/fingerprinting.py | `FingerprintingService` — compute scope fingerprints | -| local_h5_input_fingerprint | calibration/publish_local_area.py | Legacy wrapper to `FingerprintingService` | -| local_h5_resolved_geography_source | build_outputs/geography_loader.py | Resolved physical geography source | -| calibration_geography_loader | build_outputs/geography_loader.py | Resolve saved/package/legacy geography | -| load_calibration_geography | calibration/publish_local_area.py | Legacy wrapper to `CalibrationGeographyLoader` | -| geo_derive | calibration/block_assignment.py | Derive state/county/tract/PUMA/place from block GEOIDs | -| local_h5_entity_graph | build_outputs/source_dataset.py | Source-dataset entity spine | -| local_h5_microsimulation_variable_provider | build_outputs/source_dataset.py | Lazy variable accessor over source snapshot | -| local_h5_source_dataset_snapshot | build_outputs/source_dataset.py | In-memory source H5 contract | -| local_h5_policyengine_dataset_reader | build_outputs/source_dataset.py | PolicyEngine H5 adapter | -| clone_weight_matrix | build_outputs/weights.py | Shape contract for clone-level weights | -| local_h5_area_selector | build_outputs/selection.py | Apply geography filters; emit active clone rows | -| local_h5_clone_selection | build_outputs/selection.py | Selected clones + their block/CD GEOIDs | -| local_h5_entity_reindexer | build_outputs/reindexing.py | Reindex households/persons/subentities | -| local_h5_reindexed_entities | build_outputs/reindexing.py | New IDs and source-row indices | -| local_h5_variable_cloner | build_outputs/variables.py | Clone source variables into period groups | -| local_h5_variable_clone_payload | build_outputs/variables.py | Period-grouped cloned-variable payload | -| local_h5_payload | build_outputs/payload.py | Validated period-grouped H5 payload | -| local_h5_payload_build_context | build_outputs/payload.py | Context passed to postprocessors | -| local_h5_dataset_builder | build_outputs/builder.py | Coordinate select → reindex → clone → postprocess | -| local_h5_build_result | build_outputs/builder.py | In-memory build output + diagnostics | -| local_h5_us_entity_postprocessor | build_outputs/us_augmentations.py | Apply entity IDs + calibrated `household_weight` | -| local_h5_us_entity_postprocessor_result | build_outputs/us_augmentations.py | Result wrapper for entity postprocessor | -| local_h5_us_geography_postprocessor | build_outputs/us_augmentations.py | Apply state/county/tract/PUMA/etc. | -| local_h5_us_geography_postprocessor_result | build_outputs/us_augmentations.py | Result wrapper for geography postprocessor | -| local_h5_us_takeup_postprocessor | build_outputs/us_augmentations.py | Apply ACA/SNAP/etc. take-up draws | -| local_h5_us_takeup_postprocessor_result | build_outputs/us_augmentations.py | Result wrapper for take-up postprocessor | -| local_h5_validation_policy | build_outputs/validation.py | Worker-scoped validation policy | -| local_h5_validation_context | build_outputs/validation.py | Prepared per-worker validation target context | -| local_h5_area_validation_service | build_outputs/validation.py | Prepare targets once per worker session | -| local_h5_writer | build_outputs/writer.py | Write payload to disk | -| local_h5_write_result | build_outputs/writer.py | Post-write verification summary | -| build_h5 | calibration/publish_local_area.py | Transitional monolithic single-area build entry | -| build_states | calibration/publish_local_area.py | Build all state H5s with checkpointing | -| build_districts | calibration/publish_local_area.py | Build all congressional-district H5s | -| build_cities | calibration/publish_local_area.py | Build supported city H5s with county filter | -| sanity_checks | calibration/sanity_checks.py | Structural integrity checks on a written H5 | -| target_validation | calibration/validate_staging.py | Microsim target comparison for one area | -| local_stage_upload | calibration/promote_local_h5s.py | Upload built H5s to HF `staging/{run_id}/` | -| atomic_promote | calibration/promote_local_h5s.py | Promote staging to production + publish manifest + tag | diff --git a/backend/data/pipeline/stages/weight_fit.input.json b/backend/data/pipeline/stages/weight_fit.input.json deleted file mode 100644 index d50a17e..0000000 --- a/backend/data/pipeline/stages/weight_fit.input.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "stage_id": "weight_fit", - "label": "weight_fit", - "nodes": [ - { - "artifacts_in": [ - "calibration_package.pkl" - ], - "decorator_line": 696, - "description": "Build population-proportional initial weights from geography and targets.", - "id": "init_weights", - "label": "Compute Initial Weights", - "node_type": "library", - "pathways": [ - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "status": "current", - "target_symbol": "compute_initial_weights", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "artifacts_in": [ - "calibration_package.pkl" - ], - "artifacts_out": [ - "calibration_weights.npy", - "unified_diagnostics.csv" - ], - "decorator_line": 774, - "description": "Optimize sparse calibration weights with HardConcrete gates and diagnostics.", - "id": "fit_model", - "label": "Fit L0 Calibration Weights", - "node_type": "library", - "pathways": [ - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "status": "current", - "target_symbol": "fit_l0_weights", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "artifacts_out": [ - "unified_diagnostics.csv" - ], - "decorator_line": 1131, - "description": "Compare fitted weighted sums to calibration targets and summarize error.", - "id": "calibration_diagnostics", - "label": "Compute Calibration Diagnostics", - "node_type": "library", - "pathways": [ - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "status": "current", - "target_symbol": "compute_diagnostics", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - }, - { - "artifacts_in": [ - "source_imputed_stratified_extended_cps*.h5", - "policy_data.db" - ], - "artifacts_out": [ - "calibration_package.pkl", - "calibration_weights.npy", - "unified_diagnostics.csv", - "unified_run_config.json" - ], - "decorator_line": 1250, - "description": "Build or load the calibration package, fit sparse weights, and write diagnostics.", - "details": "This remains a bundled orchestration function; decorators document the semantic pathway until the implementation is further decomposed.", - "id": "run_calibration", - "label": "Run Unified Calibration", - "node_type": "entrypoint", - "pathways": [ - "calibration_package", - "weight_fit" - ], - "source_file": "policyengine_us_data/calibration/unified_calibration.py", - "stability": "moving", - "status": "transitional", - "target_symbol": "run_calibration", - "validation_commands": [ - "uv run pytest tests/unit/calibration/test_unified_calibration.py" - ] - } - ] -} \ No newline at end of file diff --git a/backend/data/pipeline/stages/weight_fit.md b/backend/data/pipeline/stages/weight_fit.md deleted file mode 100644 index a252eb1..0000000 --- a/backend/data/pipeline/stages/weight_fit.md +++ /dev/null @@ -1,68 +0,0 @@ -# Weight fit pathway - -## Purpose - -The `weight_fit` stage is the actual optimization heart of the calibration pipeline. Given a pre-built `calibration_package.pkl` (a sparse targets-by-records matrix `X_sparse`, a vector of target values, and a `targets_df` of metadata), it produces a single 1-D vector of household weights such that `X_sparse @ weights` approximates `targets` as closely as possible. It uses L0-regularized stochastic optimization (HardConcrete gates) to push as many record weights to exactly zero as it can while still matching aggregate targets — yielding a sparse, lightweight microdata file that reproduces external benchmarks (ACS, Census, IRS SOI, etc.) on each congressional district. - -## Inputs and outputs - -**In** (from `calibration_package.pkl`, built by the `calibration_package` stage): -- `X_sparse`: scipy CSR/COO matrix of shape `(n_targets, n_records)` — each row encodes how each cloned-household contributes to one target. -- `targets`: `targets_df["value"].values`, length `n_targets`. -- `targets_df`: per-target metadata (`variable`, `domain_variable`, `geo_level`, `geographic_id`, `value`); also used to derive district-level initial weights and `target_groups`. -- `target_names`, `target_groups`, `achievable` flag (from `X_sparse.sum(axis=1) > 0`). - -**Out** (written to the output directory next to the package): -- `calibration_weights.npy` — final 1-D float weight vector of length `n_records` (deterministic gate-product weights from `model.get_weights(deterministic=True)`). -- `unified_diagnostics.csv` — per-target estimate vs. target plus relative/absolute error and an `achievable` flag. -- `*.checkpoint.pt` (optional, when `checkpoint_path` is set) — resumable PyTorch state for `SparseCalibrationWeights`. -- Per-epoch `log_path` CSV (optional) — per-target estimate/error trajectory, written when `log_freq` is set. - -## Key sub-stages (in order) - -1. **`init_weights` (`compute_initial_weights`, line 712)** — Builds a population-proportional warm-start by summing `person_count` / `age` / district targets to get each CD's population, then distributing that evenly across the household columns that actually contribute to that CD; falls back to uniform `100` if no age targets exist. -2. **`fit_model` (`fit_l0_weights`, line 791)** — Instantiates `l0.calibration.SparseCalibrationWeights` with the initial weights, optimizes with Adam (`lr=LEARNING_RATE`) using a relative-error loss plus L0 + L2 penalties, logs per-epoch loss/sparsity/weight-distribution, and (optionally) writes per-target trajectories and resumable checkpoints. Returns the deterministic weight vector. -3. **`calibration_diagnostics` (`compute_diagnostics`, line 1147)** — Multiplies `X_sparse @ weights`, compares to `targets_df["value"]`, and returns a DataFrame with `target`, `true_value`, `estimate`, `rel_error`, `abs_rel_error`, and a row-sum-derived `achievable` flag (targets whose matrix row is all zeros cannot possibly be hit). -4. **`run_calibration` (line 1273)** — The transitional orchestrator. Either loads a pre-built package (early-exit path, line ~1353) or builds one in-process (steps 1-7 ending at line 1717), then calls `fit_l0_weights`, writes `calibration_weights.npy`, and runs `compute_diagnostics` to emit `unified_diagnostics.csv` plus aggregate logging. - -## What L0 regularization actually does here - -The optimization minimizes a **relative-error squared loss** (`loss_type="relative"` at line 997/1096) — i.e. for each target `i`, `((X_sparse @ w)[i] - y[i]) / |y[i]|` squared — averaged (optionally with `target_groups` to balance heterogeneous target counts). On top of that, two penalties are added: an **L0 penalty** (`lambda_l0`, default `1e-8` in `run_calibration`) and a tiny **L2 penalty** (`lambda_l2 = 1e-12`). The L0 term doesn't directly penalize the magnitude of weights; instead, `SparseCalibrationWeights` parameterizes each record's weight as `log_weight_i * gate_i`, where `gate_i` is a stochastic HardConcrete gate (Louizos et al.) with learnable `log_alpha`. The L0 penalty is the expected number of open gates, which is differentiable through the HardConcrete relaxation. This pushes the model to **literally drop records to zero weight** (gate closed) rather than just shrinking them, producing a genuinely sparse calibrated microdata file. - -The HardConcrete hyperparameters that matter: - -- **`BETA = 0.35`** — temperature of the gate distribution. Lower = harder gates (more zero-or-one), higher = softer (smoother gradients but blurrier sparsity). -- **`GAMMA = -0.1`, `ZETA = 1.1`** — the stretch interval `(γ, ζ)` of the HardConcrete; values outside `[0, 1]` are clipped, which is what makes exact zeros possible. -- **`INIT_KEEP_PROB = 0.999`** — gates start almost fully open, so the optimizer begins close to the initial weights and only sparsifies as the L0 pressure outweighs the loss benefit of keeping a record. -- **`LAMBDA_L2 = 1e-12`** — vanishing L2 on `log_weight`, mostly numerical insurance against runaway log-weights. -- **`LEARNING_RATE = 0.15`** — Adam step size; relatively large because parameters are in log-space. -- **`LOG_WEIGHT_JITTER_SD = 0.05`, `LOG_ALPHA_JITTER_SD = 0.01`** — tiny initialization noise so identical initial weights don't have identical gradients. Jitter is zeroed after the first chunk (line 946/1000) so resumed runs stay deterministic. - -## Common failure modes - -- **NaN / exploding loss** — usually a target with `|y| → 0` blowing up the relative-error denominator; the code masks `|targets| > 0` in the *diagnostic* relative error (line 1021, 1159) but the model's internal loss still sees a divide. Look at `unified_diagnostics.csv` for `true_value ≈ 0` rows and consider filtering them out of `targets_df` before building the package. -- **`unachievable` targets** — `compute_diagnostics` flags `achievable = row_sums > 0`. If a target's row in `X_sparse` is all zero (e.g. no household in the cloned dataset can possibly contribute to that geography × variable cell), the optimizer simply cannot match it. `run_calibration` logs the achievable count at line ~1712; a sudden drop usually points to a broken clone/geography assignment upstream, not a fit problem. -- **Filer-gated mismatch** — when targets reference an IRS-filer-only variable but the matrix row is built over all CPS households (or vice versa), the row sum can be non-zero but tiny, producing huge relative errors that dominate the loss and starve other targets. Check the per-target log CSV for chronically-bad targets vs. expected magnitude. -- **Tension / oscillation between targets** — if mean error stalls but max error swings each epoch, two targets share households with opposing pulls. Inspect `target_groups` weighting and consider raising `lambda_l0` (more aggressive culling) or lowering `learning_rate`. -- **Checkpoint resume rejected** — `checkpoint_signature_mismatches` (line 888) raises on structural changes (n_features, target shape). Soft mismatches (hyperparameter changes) only warn but cause transient loss/sparsity shocks for the first few epochs — expected, not a bug. -- **All weights collapse to zero (100% sparse)** — `lambda_l0` too high relative to loss; back it off by an order of magnitude. Watch the epoch logs' `active=X/N` line. - -## Why this exists / design notes - -Off-the-shelf survey calibration (raking, GREG, quadratic programming over weights directly) shrinks but never zeros weights, so the output dataset stays as large as the input. PolicyEngine clones the CPS hundreds of times to give the optimizer enough degrees of freedom to hit fine-grained district targets (`DEFAULT_N_CLONES = 430`), so without sparsification the result would be unusable in production. **L0 is preferred over L1** because L1 on weights induces a smooth shrinkage toward zero but, on a non-negative weight vector with multiplicative gates, doesn't produce a clean active/inactive partition — and L1 distorts the calibration objective (it pulls all weights down, biasing aggregates). L0 with HardConcrete gates decouples the *which records survive* decision from the *what weight do they get* decision: gates handle the former (with a sparsity prior), `log_weight` handles the latter (essentially unconstrained, modulo the negligible L2). The result is a smaller dataset whose aggregates match targets, not a shrunken dataset. - -Subtleties a reviewer should know: - -- The loss is **relative**, not absolute. A $1M target with 1% error contributes the same as a $1k target with 1% error. Without this, IRS aggregates would dominate over CPS person-counts. If you're surprised that the optimizer "ignores" a high-magnitude target, check whether it has a small but achievable row sum — relative loss will not over-weight it. -- `target_groups` (built by `create_target_groups`) further normalizes within a group so e.g. all 435 CDs' age targets are treated as one group, not 435 independent losses — preventing geographic explosion of one variable from drowning out others. -- The returned weights come from `model.get_weights(deterministic=True)` — gates are evaluated at their expected value, not sampled — so two runs from the same checkpoint give identical weights despite the stochastic gate parameterization during training. -- `init_weights` is more than a convenience: starting from population-scaled values means the optimizer mostly has to *prune* and *fine-tune*, not discover the order of magnitude from scratch. Bad initial weights will multiply training time and risk L0 culling useful records before they can demonstrate value. - -## Source map - -| Node id | Source file | Role | -|---|---|---| -| `init_weights` | `policyengine_us_data/calibration/unified_calibration.py:712` (`compute_initial_weights`) | Builds population-proportional warm-start weights from district `age` targets. | -| `fit_model` | `policyengine_us_data/calibration/unified_calibration.py:791` (`fit_l0_weights`) | Runs L0-regularized HardConcrete-gate optimization; emits `calibration_weights.npy` and per-epoch logs/checkpoints. | -| `calibration_diagnostics` | `policyengine_us_data/calibration/unified_calibration.py:1147` (`compute_diagnostics`) | Computes per-target estimate, relative error, and achievability for `unified_diagnostics.csv`. | -| `run_calibration` | `policyengine_us_data/calibration/unified_calibration.py:1273` (`run_calibration`) | Transitional orchestrator: build-or-load package, call `fit_l0_weights` (line 1369 or 1717), write weights, then `compute_diagnostics` (line 1884). | diff --git a/backend/data/target_index.json b/backend/data/target_index.json deleted file mode 100644 index c1b12b3..0000000 --- a/backend/data/target_index.json +++ /dev/null @@ -1,4217 +0,0 @@ -{ - "db_total": 37812, - "tiers": [ - { - "tier": "csv/aca_marketplace_state_metal_selection_2024.csv", - "total_records": 192, - "unique_signatures": 192, - "matched_to_db": 0, - "unmatched": 192, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "aca_metal_level", - "==", - "bronze" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 11197.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv", - "source_row": "row-5", - "notes": "CMS 2024 OEP PUF \u2014 marketplace consumers, metal=bronze", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "aca_metal_level", - "==", - "bronze" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "aca_metal_level", - "==", - "bronze" - ], - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 8733.66, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv", - "source_row": "row-5", - "notes": "CMS 2024 OEP PUF \u2014 APTC consumers, metal=bronze", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "aca_metal_level", - "==", - "bronze" - ], - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "aca_metal_level", - "==", - "gold" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 12546.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv", - "source_row": "row-6", - "notes": "CMS 2024 OEP PUF \u2014 marketplace consumers, metal=gold", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "aca_metal_level", - "==", - "gold" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "aca_metal_level", - "==", - "gold" - ], - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 11165.94, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv", - "source_row": "row-6", - "notes": "CMS 2024 OEP PUF \u2014 APTC consumers, metal=gold", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "aca_metal_level", - "==", - "gold" - ], - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "aca_metal_level", - "==", - "silver" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 3721.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv", - "source_row": "row-7", - "notes": "CMS 2024 OEP PUF \u2014 marketplace consumers, metal=silver", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "aca_metal_level", - "==", - "silver" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/aca_ptc_state.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 51, - "unmatched": 51, - "match_rate": 0.5, - "unmatched_examples": [ - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "1", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 1206545000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_ptc_state.csv", - "source_row": "row-2", - "notes": "IRS SOI Historical Table 2 \u2014 PTC amount (A85770, $)", - "signature": [ - "aca_ptc", - "state", - "1", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "2", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 119862000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_ptc_state.csv", - "source_row": "row-3", - "notes": "IRS SOI Historical Table 2 \u2014 PTC amount (A85770, $)", - "signature": [ - "aca_ptc", - "state", - "2", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "4", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 644411000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_ptc_state.csv", - "source_row": "row-4", - "notes": "IRS SOI Historical Table 2 \u2014 PTC amount (A85770, $)", - "signature": [ - "aca_ptc", - "state", - "4", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "5", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 293948000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_ptc_state.csv", - "source_row": "row-5", - "notes": "IRS SOI Historical Table 2 \u2014 PTC amount (A85770, $)", - "signature": [ - "aca_ptc", - "state", - "5", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "6", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 6379623000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_ptc_state.csv", - "source_row": "row-6", - "notes": "IRS SOI Historical Table 2 \u2014 PTC amount (A85770, $)", - "signature": [ - "aca_ptc", - "state", - "6", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/aca_spending_and_enrollment_2024.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 0, - "unmatched": 102, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 23610.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2024.csv", - "source_row": "row-2", - "notes": "ACA marketplace enrollment (PTC recipients), 2024", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 20422650.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2024.csv", - "source_row": "row-2", - "notes": "ACA PTC spending ($), 2024", - "signature": [ - "aca_ptc", - "state", - "2", - 2024, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 371423.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2024.csv", - "source_row": "row-3", - "notes": "ACA marketplace enrollment (PTC recipients), 2024", - "signature": [ - "tax_unit_count", - "state", - "1", - 2024, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 243653488.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2024.csv", - "source_row": "row-3", - "notes": "ACA PTC spending ($), 2024", - "signature": [ - "aca_ptc", - "state", - "1", - 2024, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "5", - "period": 2024, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 144896.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2024.csv", - "source_row": "row-4", - "notes": "ACA marketplace enrollment (PTC recipients), 2024", - "signature": [ - "tax_unit_count", - "state", - "5", - 2024, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/aca_spending_and_enrollment_2025.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 0, - "unmatched": 102, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2025, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 24249.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2025.csv", - "source_row": "row-2", - "notes": "ACA marketplace enrollment (PTC recipients), 2025", - "signature": [ - "tax_unit_count", - "state", - "2", - 2025, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "2", - "period": 2025, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 24458996.34, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2025.csv", - "source_row": "row-2", - "notes": "ACA PTC spending ($), 2025", - "signature": [ - "aca_ptc", - "state", - "2", - 2025, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2025, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 446956.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2025.csv", - "source_row": "row-3", - "notes": "ACA marketplace enrollment (PTC recipients), 2025", - "signature": [ - "tax_unit_count", - "state", - "1", - 2025, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "1", - "period": 2025, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 273304654.88, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2025.csv", - "source_row": "row-3", - "notes": "ACA PTC spending ($), 2025", - "signature": [ - "aca_ptc", - "state", - "1", - 2025, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "5", - "period": 2025, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 143734.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2025.csv", - "source_row": "row-4", - "notes": "ACA marketplace enrollment (PTC recipients), 2025", - "signature": [ - "tax_unit_count", - "state", - "5", - 2025, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/aca_spending_and_enrollment_2026.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 0, - "unmatched": 102, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2026, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 19249.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2026.csv", - "source_row": "row-2", - "notes": "ACA marketplace enrollment (PTC recipients), 2026", - "signature": [ - "tax_unit_count", - "state", - "2", - 2026, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "2", - "period": 2026, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 18228803.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2026.csv", - "source_row": "row-2", - "notes": "ACA PTC spending ($), 2026", - "signature": [ - "aca_ptc", - "state", - "2", - 2026, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2026, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 415172.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2026.csv", - "source_row": "row-3", - "notes": "ACA marketplace enrollment (PTC recipients), 2026", - "signature": [ - "tax_unit_count", - "state", - "1", - 2026, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "aca_ptc", - "geo_level": "state", - "geographic_id": "1", - "period": 2026, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 281486616.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2026.csv", - "source_row": "row-3", - "notes": "ACA PTC spending ($), 2026", - "signature": [ - "aca_ptc", - "state", - "1", - 2026, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "5", - "period": 2026, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 139372.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/aca_spending_and_enrollment_2026.csv", - "source_row": "row-4", - "notes": "ACA marketplace enrollment (PTC recipients), 2026", - "signature": [ - "tax_unit_count", - "state", - "5", - 2026, - [ - [ - "aca_ptc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/acs_housing_costs_2024.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 0, - "unmatched": 102, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "rent", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [], - "value": 1350681600.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/acs_housing_costs_2024.csv", - "source_row": "row-2/rent", - "notes": "ACS 2024 \u2014 annual contract rent (state aggregate, $)", - "signature": [ - "rent", - "state", - "2", - 2024, - [] - ] - }, - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [], - "value": 664772900.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/acs_housing_costs_2024.csv", - "source_row": "row-2/real_estate_taxes", - "notes": "ACS 2024 \u2014 real estate taxes (state aggregate, $)", - "signature": [ - "real_estate_taxes", - "state", - "2", - 2024, - [] - ] - }, - { - "variable": "rent", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [], - "value": 5761773600.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/acs_housing_costs_2024.csv", - "source_row": "row-3/rent", - "notes": "ACS 2024 \u2014 annual contract rent (state aggregate, $)", - "signature": [ - "rent", - "state", - "1", - 2024, - [] - ] - }, - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [], - "value": 1537253700.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/acs_housing_costs_2024.csv", - "source_row": "row-3/real_estate_taxes", - "notes": "ACS 2024 \u2014 real estate taxes (state aggregate, $)", - "signature": [ - "real_estate_taxes", - "state", - "1", - 2024, - [] - ] - }, - { - "variable": "rent", - "geo_level": "state", - "geographic_id": "5", - "period": 2024, - "constraints": [], - "value": 3760575600.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/acs_housing_costs_2024.csv", - "source_row": "row-4/rent", - "notes": "ACS 2024 \u2014 annual contract rent (state aggregate, $)", - "signature": [ - "rent", - "state", - "5", - 2024, - [] - ] - } - ] - }, - { - "tier": "csv/age_state.csv", - "total_records": 900, - "unique_signatures": 900, - "matched_to_db": 0, - "unmatched": 900, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "0" - ], - [ - "age", - "<", - "5" - ] - ], - "value": 288019.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/age_state.csv", - "source_row": "row-2/0-4", - "notes": "Census ACS age \u00d7 state \u2014 band 0-4", - "signature": [ - "person_count", - "state", - "1", - 2024, - [ - [ - "age", - "<", - "5" - ], - [ - "age", - ">=", - "0" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "5" - ], - [ - "age", - "<", - "10" - ] - ], - "value": 305731.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/age_state.csv", - "source_row": "row-2/5-9", - "notes": "Census ACS age \u00d7 state \u2014 band 5-9", - "signature": [ - "person_count", - "state", - "1", - 2024, - [ - [ - "age", - "<", - "10" - ], - [ - "age", - ">=", - "5" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "10" - ], - [ - "age", - "<", - "15" - ] - ], - "value": 331262.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/age_state.csv", - "source_row": "row-2/10-14", - "notes": "Census ACS age \u00d7 state \u2014 band 10-14", - "signature": [ - "person_count", - "state", - "1", - 2024, - [ - [ - "age", - "<", - "15" - ], - [ - "age", - ">=", - "10" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "15" - ], - [ - "age", - "<", - "20" - ] - ], - "value": 350694.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/age_state.csv", - "source_row": "row-2/15-19", - "notes": "Census ACS age \u00d7 state \u2014 band 15-19", - "signature": [ - "person_count", - "state", - "1", - 2024, - [ - [ - "age", - "<", - "20" - ], - [ - "age", - ">=", - "15" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "20" - ], - [ - "age", - "<", - "25" - ] - ], - "value": 333795.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/age_state.csv", - "source_row": "row-2/20-24", - "notes": "Census ACS age \u00d7 state \u2014 band 20-24", - "signature": [ - "person_count", - "state", - "1", - 2024, - [ - [ - "age", - "<", - "25" - ], - [ - "age", - ">=", - "20" - ] - ] - ] - } - ] - }, - { - "tier": "csv/agi_state.csv", - "total_records": 1020, - "unique_signatures": 1020, - "matched_to_db": 0, - "unmatched": 1020, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - "<", - "1.0" - ] - ], - "value": 32590.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/agi_state.csv", - "source_row": "0400000US01/adjusted_gross_income/count/-inf-1.0#0", - "notes": "IRS-SOI state AGI band (adjusted_gross_income/count)", - "signature": [ - "tax_unit_count", - "state", - "1", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "1" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - ">=", - "1.0" - ], - [ - "adjusted_gross_income", - "<", - "10000.0" - ] - ], - "value": 219420.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/agi_state.csv", - "source_row": "0400000US01/adjusted_gross_income/count/1.0-10000.0#1", - "notes": "IRS-SOI state AGI band (adjusted_gross_income/count)", - "signature": [ - "tax_unit_count", - "state", - "1", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "10000" - ], - [ - "adjusted_gross_income", - ">=", - "1" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - ">=", - "10000.0" - ], - [ - "adjusted_gross_income", - "<", - "25000.0" - ] - ], - "value": 440290.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/agi_state.csv", - "source_row": "0400000US01/adjusted_gross_income/count/10000.0-25000.0#2", - "notes": "IRS-SOI state AGI band (adjusted_gross_income/count)", - "signature": [ - "tax_unit_count", - "state", - "1", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "25000" - ], - [ - "adjusted_gross_income", - ">=", - "10000" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - ">=", - "25000.0" - ], - [ - "adjusted_gross_income", - "<", - "50000.0" - ] - ], - "value": 533690.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/agi_state.csv", - "source_row": "0400000US01/adjusted_gross_income/count/25000.0-50000.0#3", - "notes": "IRS-SOI state AGI band (adjusted_gross_income/count)", - "signature": [ - "tax_unit_count", - "state", - "1", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "50000" - ], - [ - "adjusted_gross_income", - ">=", - "25000" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - ">=", - "50000.0" - ], - [ - "adjusted_gross_income", - "<", - "75000.0" - ] - ], - "value": 304710.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/agi_state.csv", - "source_row": "0400000US01/adjusted_gross_income/count/50000.0-75000.0#4", - "notes": "IRS-SOI state AGI band (adjusted_gross_income/count)", - "signature": [ - "tax_unit_count", - "state", - "1", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "75000" - ], - [ - "adjusted_gross_income", - ">=", - "50000" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/eitc_by_agi_and_children.csv", - "total_records": 224, - "unique_signatures": 224, - "matched_to_db": 0, - "unmatched": 224, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "adjusted_gross_income", - "<", - "1.0" - ] - ], - "value": 97411.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_by_agi_and_children.csv", - "source_row": "c0/-inf-1#0", - "notes": "IRS SOI Pub 1304 Table 2.5 (TY2022) \u2014 EITC returns by AGI\u00d7kids", - "signature": [ - "tax_unit_count", - "national", - "", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "eitc", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "adjusted_gross_income", - "<", - "1.0" - ] - ], - "value": 41206000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_by_agi_and_children.csv", - "source_row": "c0/-inf-1#0", - "notes": "IRS SOI Pub 1304 Table 2.5 (TY2022) \u2014 EITC amount by AGI\u00d7kids", - "signature": [ - "eitc", - "national", - "", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "adjusted_gross_income", - ">=", - "1.0" - ], - [ - "adjusted_gross_income", - "<", - "1000.0" - ] - ], - "value": 285178.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_by_agi_and_children.csv", - "source_row": "c0/1-1000#1", - "notes": "IRS SOI Pub 1304 Table 2.5 (TY2022) \u2014 EITC returns by AGI\u00d7kids", - "signature": [ - "tax_unit_count", - "national", - "", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "1000" - ], - [ - "adjusted_gross_income", - ">=", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "eitc", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "adjusted_gross_income", - ">=", - "1.0" - ], - [ - "adjusted_gross_income", - "<", - "1000.0" - ] - ], - "value": 22736000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_by_agi_and_children.csv", - "source_row": "c0/1-1000#1", - "notes": "IRS SOI Pub 1304 Table 2.5 (TY2022) \u2014 EITC amount by AGI\u00d7kids", - "signature": [ - "eitc", - "national", - "", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "1000" - ], - [ - "adjusted_gross_income", - ">=", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "adjusted_gross_income", - ">=", - "1000.0" - ], - [ - "adjusted_gross_income", - "<", - "2000.0" - ] - ], - "value": 281955.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_by_agi_and_children.csv", - "source_row": "c0/1000-2000#2", - "notes": "IRS SOI Pub 1304 Table 2.5 (TY2022) \u2014 EITC returns by AGI\u00d7kids", - "signature": [ - "tax_unit_count", - "national", - "", - 2022, - [ - [ - "adjusted_gross_income", - "<", - "2000" - ], - [ - "adjusted_gross_income", - ">=", - "1000" - ], - [ - "eitc", - ">", - "0" - ], - [ - "eitc_child_count", - "==", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/eitc_claim_controls.csv", - "total_records": 106, - "unique_signatures": 104, - "matched_to_db": 2, - "unmatched": 104, - "match_rate": 0.018867924528301886, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 24000000.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_claim_controls.csv", - "source_row": "0100000US#2024#0", - "notes": "IRS EITC claim controls TY2024 \u2014 returns", - "signature": [ - "tax_unit_count", - "national", - "", - 2024, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 10000.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_claim_controls.csv", - "source_row": "INTL#2024#52", - "notes": "IRS EITC claim controls TY2024 \u2014 returns", - "signature": [ - "tax_unit_count", - "national", - "", - 2024, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 430600.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_claim_controls.csv", - "source_row": "0400000US01#2024#1", - "notes": "IRS EITC claim controls TY2024 \u2014 returns", - "signature": [ - "tax_unit_count", - "state", - "1", - 2024, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "eitc", - "geo_level": "state", - "geographic_id": "01", - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 1400000000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_claim_controls.csv", - "source_row": "0400000US01#2024#1", - "notes": "IRS EITC claim controls TY2024 \u2014 amount", - "signature": [ - "eitc", - "state", - "1", - 2024, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "02", - "period": 2024, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 37800.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_claim_controls.csv", - "source_row": "0400000US02#2024#2", - "notes": "IRS EITC claim controls TY2024 \u2014 returns", - "signature": [ - "tax_unit_count", - "state", - "2", - 2024, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/eitc_state.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 0, - "unmatched": 102, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 440510.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_state.csv", - "source_row": "0400000US01#0", - "notes": "IRS SOI Historical Table 2 (TY2022) \u2014 EITC returns", - "signature": [ - "tax_unit_count", - "state", - "1", - 2022, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "eitc", - "geo_level": "state", - "geographic_id": "01", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 1262145000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_state.csv", - "source_row": "0400000US01#0", - "notes": "IRS SOI Historical Table 2 (TY2022) \u2014 EITC amount", - "signature": [ - "eitc", - "state", - "1", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "02", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 38650.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_state.csv", - "source_row": "0400000US02#1", - "notes": "IRS SOI Historical Table 2 (TY2022) \u2014 EITC returns", - "signature": [ - "tax_unit_count", - "state", - "2", - 2022, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "eitc", - "geo_level": "state", - "geographic_id": "02", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ], - "value": 82237000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_state.csv", - "source_row": "0400000US02#1", - "notes": "IRS SOI Historical Table 2 (TY2022) \u2014 EITC amount", - "signature": [ - "eitc", - "state", - "2", - 2022, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "state", - "geographic_id": "04", - "period": 2022, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "eitc", - ">", - "0" - ] - ], - "value": 515920.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/eitc_state.csv", - "source_row": "0400000US04#2", - "notes": "IRS SOI Historical Table 2 (TY2022) \u2014 EITC returns", - "signature": [ - "tax_unit_count", - "state", - "4", - 2022, - [ - [ - "eitc", - ">", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/healthcare_spending.csv", - "total_records": 36, - "unique_signatures": 36, - "matched_to_db": 0, - "unmatched": 36, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "health_insurance_premiums_without_medicare_part_b", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "0" - ], - [ - "age", - "<", - "10" - ] - ], - "value": 0.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/healthcare_spending.csv", - "source_row": "row-2/band-0", - "notes": "Healthcare spending 2024 \u00b7 age band 0+", - "signature": [ - "health_insurance_premiums_without_medicare_part_b", - "national", - "", - 2024, - [ - [ - "age", - "<", - "10" - ], - [ - "age", - ">=", - "0" - ] - ] - ] - }, - { - "variable": "over_the_counter_health_expenses", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "0" - ], - [ - "age", - "<", - "10" - ] - ], - "value": 3139958418.4186344, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/healthcare_spending.csv", - "source_row": "row-2/band-0", - "notes": "Healthcare spending 2024 \u00b7 age band 0+", - "signature": [ - "over_the_counter_health_expenses", - "national", - "", - 2024, - [ - [ - "age", - "<", - "10" - ], - [ - "age", - ">=", - "0" - ] - ] - ] - }, - { - "variable": "other_medical_expenses", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "0" - ], - [ - "age", - "<", - "10" - ] - ], - "value": 10581757838.419937, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/healthcare_spending.csv", - "source_row": "row-2/band-0", - "notes": "Healthcare spending 2024 \u00b7 age band 0+", - "signature": [ - "other_medical_expenses", - "national", - "", - 2024, - [ - [ - "age", - "<", - "10" - ], - [ - "age", - ">=", - "0" - ] - ] - ] - }, - { - "variable": "medicare_part_b_premiums", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "0" - ], - [ - "age", - "<", - "10" - ] - ], - "value": 0.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/healthcare_spending.csv", - "source_row": "row-2/band-0", - "notes": "Healthcare spending 2024 \u00b7 age band 0+", - "signature": [ - "medicare_part_b_premiums", - "national", - "", - 2024, - [ - [ - "age", - "<", - "10" - ], - [ - "age", - ">=", - "0" - ] - ] - ] - }, - { - "variable": "health_insurance_premiums_without_medicare_part_b", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "age", - ">=", - "10" - ], - [ - "age", - "<", - "20" - ] - ], - "value": 1331908850.7960727, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/healthcare_spending.csv", - "source_row": "row-3/band-10", - "notes": "Healthcare spending 2024 \u00b7 age band 10+", - "signature": [ - "health_insurance_premiums_without_medicare_part_b", - "national", - "", - 2024, - [ - [ - "age", - "<", - "20" - ], - [ - "age", - ">=", - "10" - ] - ] - ] - } - ] - }, - { - "tier": "csv/medicaid_enrollment_2024.csv", - "total_records": 51, - "unique_signatures": 51, - "matched_to_db": 0, - "unmatched": 51, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 231577.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2024.csv", - "source_row": "row-2", - "notes": "Medicaid enrollment 2024 \u2014 AK (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "2", - 2024, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 766009.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2024.csv", - "source_row": "row-3", - "notes": "Medicaid enrollment 2024 \u2014 AL (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "1", - 2024, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "5", - "period": 2024, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 733561.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2024.csv", - "source_row": "row-4", - "notes": "Medicaid enrollment 2024 \u2014 AR (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "5", - 2024, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "4", - "period": 2024, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 1778734.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2024.csv", - "source_row": "row-5", - "notes": "Medicaid enrollment 2024 \u2014 AZ (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "4", - 2024, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "6", - "period": 2024, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 12172695.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2024.csv", - "source_row": "row-6", - "notes": "Medicaid enrollment 2024 \u2014 CA (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "6", - 2024, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/medicaid_enrollment_2025.csv", - "total_records": 51, - "unique_signatures": 51, - "matched_to_db": 0, - "unmatched": 51, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2025, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 199460.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2025.csv", - "source_row": "row-2", - "notes": "Medicaid enrollment 2025 \u2014 AK (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "2", - 2025, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2025, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 752535.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2025.csv", - "source_row": "row-3", - "notes": "Medicaid enrollment 2025 \u2014 AL (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "1", - 2025, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "5", - "period": 2025, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 723536.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2025.csv", - "source_row": "row-4", - "notes": "Medicaid enrollment 2025 \u2014 AR (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "5", - 2025, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "4", - "period": 2025, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 1579905.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2025.csv", - "source_row": "row-5", - "notes": "Medicaid enrollment 2025 \u2014 AZ (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "4", - 2025, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "6", - "period": 2025, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 11554412.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2025.csv", - "source_row": "row-6", - "notes": "Medicaid enrollment 2025 \u2014 CA (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "6", - 2025, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/medicaid_enrollment_2026.csv", - "total_records": 51, - "unique_signatures": 51, - "matched_to_db": 0, - "unmatched": 51, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "2", - "period": 2026, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 197303.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2026.csv", - "source_row": "row-2", - "notes": "Medicaid enrollment 2026 \u2014 AK (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "2", - 2026, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "1", - "period": 2026, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 741054.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2026.csv", - "source_row": "row-3", - "notes": "Medicaid enrollment 2026 \u2014 AL (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "1", - 2026, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "5", - "period": 2026, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 717460.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2026.csv", - "source_row": "row-4", - "notes": "Medicaid enrollment 2026 \u2014 AR (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "5", - 2026, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "4", - "period": 2026, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 1550145.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2026.csv", - "source_row": "row-5", - "notes": "Medicaid enrollment 2026 \u2014 AZ (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "4", - 2026, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "6", - "period": 2026, - "constraints": [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ], - "value": 11317458.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/medicaid_enrollment_2026.csv", - "source_row": "row-6", - "notes": "Medicaid enrollment 2026 \u2014 CA (persons with medicaid_enrolled==1)", - "signature": [ - "person_count", - "state", - "6", - 2026, - [ - [ - "medicaid_enrolled", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/np2023_d5_mid.csv", - "total_records": 6873, - "unique_signatures": 6873, - "matched_to_db": 0, - "unmatched": 6873, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [], - "value": 286803769.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/np2023_d5_mid.csv", - "source_row": "row-2/year-2022/total", - "notes": "Census NP2023 mid-series \u2014 total US population", - "signature": [ - "person_count", - "national", - "", - 2022, - [] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "age", - "==", - "0" - ] - ], - "value": 3623265.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/np2023_d5_mid.csv", - "source_row": "row-2/year-2022/POP_0", - "notes": "Census NP2023 mid-series \u2014 age 0", - "signature": [ - "person_count", - "national", - "", - 2022, - [ - [ - "age", - "==", - "0" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "age", - "==", - "1" - ] - ], - "value": 3523396.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/np2023_d5_mid.csv", - "source_row": "row-2/year-2022/POP_1", - "notes": "Census NP2023 mid-series \u2014 age 1", - "signature": [ - "person_count", - "national", - "", - 2022, - [ - [ - "age", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "age", - "==", - "2" - ] - ], - "value": 3619958.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/np2023_d5_mid.csv", - "source_row": "row-2/year-2022/POP_2", - "notes": "Census NP2023 mid-series \u2014 age 2", - "signature": [ - "person_count", - "national", - "", - 2022, - [ - [ - "age", - "==", - "2" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2022, - "constraints": [ - [ - "age", - "==", - "3" - ] - ], - "value": 3661065.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/np2023_d5_mid.csv", - "source_row": "row-2/year-2022/POP_3", - "notes": "Census NP2023 mid-series \u2014 age 3", - "signature": [ - "person_count", - "national", - "", - 2022, - [ - [ - "age", - "==", - "3" - ] - ] - ] - } - ] - }, - { - "tier": "csv/population_by_state.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 0, - "unmatched": 102, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "6", - "period": 2024, - "constraints": [], - "value": 38965193.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/population_by_state.csv", - "source_row": "row-2/total", - "notes": "Census state population \u2014 total", - "signature": [ - "person_count", - "state", - "6", - 2024, - [] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "6", - "period": 2024, - "constraints": [ - [ - "age", - "<", - "5" - ] - ], - "value": 2104120.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/population_by_state.csv", - "source_row": "row-2/under_5", - "notes": "Census state population \u2014 under 5", - "signature": [ - "person_count", - "state", - "6", - 2024, - [ - [ - "age", - "<", - "5" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "48", - "period": 2024, - "constraints": [], - "value": 30503301.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/population_by_state.csv", - "source_row": "row-3/total", - "notes": "Census state population \u2014 total", - "signature": [ - "person_count", - "state", - "48", - 2024, - [] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "48", - "period": 2024, - "constraints": [ - [ - "age", - "<", - "5" - ] - ], - "value": 1921708.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/population_by_state.csv", - "source_row": "row-3/under_5", - "notes": "Census state population \u2014 under 5", - "signature": [ - "person_count", - "state", - "48", - 2024, - [ - [ - "age", - "<", - "5" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "state", - "geographic_id": "12", - "period": 2024, - "constraints": [], - "value": 22610726.0, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/population_by_state.csv", - "source_row": "row-4/total", - "notes": "Census state population \u2014 total", - "signature": [ - "person_count", - "state", - "12", - 2024, - [] - ] - } - ] - }, - { - "tier": "csv/real_estate_taxes_by_state_acs.csv", - "total_records": 51, - "unique_signatures": 51, - "matched_to_db": 0, - "unmatched": 51, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "2", - "period": 2024, - "constraints": [], - "value": 88000000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/real_estate_taxes_by_state_acs.csv", - "source_row": "row-2", - "notes": "ACS \u2014 real estate taxes by state ($, scaled \u00d71e9 from billions)", - "signature": [ - "real_estate_taxes", - "state", - "2", - 2024, - [] - ] - }, - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "1", - "period": 2024, - "constraints": [], - "value": 616000000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/real_estate_taxes_by_state_acs.csv", - "source_row": "row-3", - "notes": "ACS \u2014 real estate taxes by state ($, scaled \u00d71e9 from billions)", - "signature": [ - "real_estate_taxes", - "state", - "1", - 2024, - [] - ] - }, - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "5", - "period": 2024, - "constraints": [], - "value": 785000000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/real_estate_taxes_by_state_acs.csv", - "source_row": "row-4", - "notes": "ACS \u2014 real estate taxes by state ($, scaled \u00d71e9 from billions)", - "signature": [ - "real_estate_taxes", - "state", - "5", - 2024, - [] - ] - }, - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "4", - "period": 2024, - "constraints": [], - "value": 1355000000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/real_estate_taxes_by_state_acs.csv", - "source_row": "row-5", - "notes": "ACS \u2014 real estate taxes by state ($, scaled \u00d71e9 from billions)", - "signature": [ - "real_estate_taxes", - "state", - "4", - 2024, - [] - ] - }, - { - "variable": "real_estate_taxes", - "geo_level": "state", - "geographic_id": "6", - "period": 2024, - "constraints": [], - "value": 11879000000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/real_estate_taxes_by_state_acs.csv", - "source_row": "row-6", - "notes": "ACS \u2014 real estate taxes by state ($, scaled \u00d71e9 from billions)", - "signature": [ - "real_estate_taxes", - "state", - "6", - 2024, - [] - ] - } - ] - }, - { - "tier": "csv/snap_state.csv", - "total_records": 102, - "unique_signatures": 102, - "matched_to_db": 51, - "unmatched": 51, - "match_rate": 0.5, - "unmatched_examples": [ - { - "variable": "snap", - "geo_level": "state", - "geographic_id": "01", - "period": 2024, - "constraints": [], - "value": 1733693703.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/snap_state.csv", - "source_row": "row-2", - "notes": "USDA FNS SNAP \u2014 dollar cost", - "signature": [ - "snap", - "state", - "1", - 2024, - [] - ] - }, - { - "variable": "snap", - "geo_level": "state", - "geographic_id": "02", - "period": 2024, - "constraints": [], - "value": 249618195.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/snap_state.csv", - "source_row": "row-3", - "notes": "USDA FNS SNAP \u2014 dollar cost", - "signature": [ - "snap", - "state", - "2", - 2024, - [] - ] - }, - { - "variable": "snap", - "geo_level": "state", - "geographic_id": "04", - "period": 2024, - "constraints": [], - "value": 2015194104.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/snap_state.csv", - "source_row": "row-4", - "notes": "USDA FNS SNAP \u2014 dollar cost", - "signature": [ - "snap", - "state", - "4", - 2024, - [] - ] - }, - { - "variable": "snap", - "geo_level": "state", - "geographic_id": "05", - "period": 2024, - "constraints": [], - "value": 549666027.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/snap_state.csv", - "source_row": "row-5", - "notes": "USDA FNS SNAP \u2014 dollar cost", - "signature": [ - "snap", - "state", - "5", - 2024, - [] - ] - }, - { - "variable": "snap", - "geo_level": "state", - "geographic_id": "06", - "period": 2024, - "constraints": [], - "value": 12377175489.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/snap_state.csv", - "source_row": "row-6", - "notes": "USDA FNS SNAP \u2014 dollar cost", - "signature": [ - "snap", - "state", - "6", - 2024, - [] - ] - } - ] - }, - { - "tier": "csv/soi_targets.csv", - "total_records": 11929, - "unique_signatures": 6748, - "matched_to_db": 0, - "unmatched": 11929, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2015, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - "<", - "0.0" - ] - ], - "value": -203775058000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/soi_targets.csv", - "source_row": "Table 1.1/11/D", - "notes": "IRS SOI Table 1.1 \u00b7 filing=All", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2015, - [ - [ - "adjusted_gross_income", - "<", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2015, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - "<", - "0.0" - ], - [ - "tax_unit_is_taxable", - "==", - "1" - ] - ], - "value": -11205340000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/soi_targets.csv", - "source_row": "Table 1.1/11/I", - "notes": "IRS SOI Table 1.1 \u00b7 filing=All", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2015, - [ - [ - "adjusted_gross_income", - "<", - "0" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "tax_unit_is_taxable", - "==", - "1" - ] - ] - ] - }, - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2015, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "full_population", - "==", - "1" - ] - ], - "value": 10210310102000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/soi_targets.csv", - "source_row": "Table 1.1/10/D", - "notes": "IRS SOI Table 1.1 \u00b7 filing=All", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2015, - [ - [ - "full_population", - "==", - "1" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - }, - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2015, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "tax_unit_is_taxable", - "==", - "1" - ] - ], - "value": 9550843480000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/soi_targets.csv", - "source_row": "Table 1.1/10/I", - "notes": "IRS SOI Table 1.1 \u00b7 filing=All", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2015, - [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "tax_unit_is_taxable", - "==", - "1" - ] - ] - ] - }, - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2015, - "constraints": [ - [ - "tax_unit_is_filer", - "==", - "1" - ], - [ - "adjusted_gross_income", - ">=", - "1.0" - ], - [ - "adjusted_gross_income", - "<", - "5000.0" - ] - ], - "value": 26240797000.0, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/soi_targets.csv", - "source_row": "Table 1.2/11/C", - "notes": "IRS SOI Table 1.2 \u00b7 filing=All", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2015, - [ - [ - "adjusted_gross_income", - "<", - "5000" - ], - [ - "adjusted_gross_income", - ">=", - "1" - ], - [ - "tax_unit_is_filer", - "==", - "1" - ] - ] - ] - } - ] - }, - { - "tier": "csv/spm_threshold_agi.csv", - "total_records": 20, - "unique_signatures": 20, - "matched_to_db": 0, - "unmatched": 20, - "match_rate": 0.0, - "unmatched_examples": [ - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "spm_unit_spm_threshold", - ">=", - "13232.297" - ], - [ - "spm_unit_spm_threshold", - "<", - "16084.154296875" - ] - ], - "value": 601395945534.2526, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/spm_threshold_agi.csv", - "source_row": "row-2/decile-1", - "notes": "SPM-threshold decile 1 \u00b7 AGI dollar amount", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2024, - [ - [ - "spm_unit_spm_threshold", - "<", - "16084" - ], - [ - "spm_unit_spm_threshold", - ">=", - "13232" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "spm_unit_spm_threshold", - ">=", - "13232.297" - ], - [ - "spm_unit_spm_threshold", - "<", - "16084.154296875" - ] - ], - "value": 14067622.329864502, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/spm_threshold_agi.csv", - "source_row": "row-2/decile-1", - "notes": "SPM-threshold decile 1 \u00b7 tax-unit count", - "signature": [ - "tax_unit_count", - "national", - "", - 2024, - [ - [ - "spm_unit_spm_threshold", - "<", - "16084" - ], - [ - "spm_unit_spm_threshold", - ">=", - "13232" - ] - ] - ] - }, - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "spm_unit_spm_threshold", - ">=", - "16084.154" - ], - [ - "spm_unit_spm_threshold", - "<", - "18203.76171875" - ] - ], - "value": 699173775752.6232, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/spm_threshold_agi.csv", - "source_row": "row-3/decile-2", - "notes": "SPM-threshold decile 2 \u00b7 AGI dollar amount", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2024, - [ - [ - "spm_unit_spm_threshold", - "<", - "18203" - ], - [ - "spm_unit_spm_threshold", - ">=", - "16084" - ] - ] - ] - }, - { - "variable": "tax_unit_count", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "spm_unit_spm_threshold", - ">=", - "16084.154" - ], - [ - "spm_unit_spm_threshold", - "<", - "18203.76171875" - ] - ], - "value": 14069629.232849121, - "is_count": true, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/spm_threshold_agi.csv", - "source_row": "row-3/decile-2", - "notes": "SPM-threshold decile 2 \u00b7 tax-unit count", - "signature": [ - "tax_unit_count", - "national", - "", - 2024, - [ - [ - "spm_unit_spm_threshold", - "<", - "18203" - ], - [ - "spm_unit_spm_threshold", - ">=", - "16084" - ] - ] - ] - }, - { - "variable": "adjusted_gross_income", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "spm_unit_spm_threshold", - ">=", - "18203.762" - ], - [ - "spm_unit_spm_threshold", - "<", - "19966.45703125" - ] - ], - "value": 1038943520151.5376, - "is_count": false, - "storage_tier": "csv", - "source_path": "storage/calibration_targets/spm_threshold_agi.csv", - "source_row": "row-4/decile-3", - "notes": "SPM-threshold decile 3 \u00b7 AGI dollar amount", - "signature": [ - "adjusted_gross_income", - "national", - "", - 2024, - [ - [ - "spm_unit_spm_threshold", - "<", - "19966" - ], - [ - "spm_unit_spm_threshold", - ">=", - "18203" - ] - ] - ] - } - ] - }, - { - "tier": "python/cms_medicare+takeup", - "total_records": 5, - "unique_signatures": 5, - "matched_to_db": 1, - "unmatched": 4, - "match_rate": 0.2, - "unmatched_examples": [ - { - "variable": "medicare_part_b", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [], - "value": 139837000000.0, - "is_count": false, - "storage_tier": "python", - "source_path": "utils/cms_medicare.py", - "source_row": "MEDICARE_PART_B_GROSS_PREMIUM_INCOME", - "notes": "CMS Medicare Trustees Report \u2014 gross Part B premium income", - "signature": [ - "medicare_part_b", - "national", - "", - 2024, - [] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "medicare_enrolled", - "==", - "1" - ] - ], - "value": 68030000.0, - "is_count": true, - "storage_tier": "python", - "source_path": "utils/cms_medicare.py", - "source_row": "MEDICARE_ENROLLMENT_TARGETS", - "notes": "CMS Medicare Trustees Report Table V.B3 \u2014 enrollee count", - "signature": [ - "person_count", - "national", - "", - 2024, - [ - [ - "medicare_enrolled", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2024, - "constraints": [ - [ - "state_buy_in_medicare", - "==", - "1" - ] - ], - "value": 10000000.0, - "is_count": true, - "storage_tier": "python", - "source_path": "utils/cms_medicare.py", - "source_row": "MEDICARE_STATE_BUY_IN_MINIMUM_BENEFICIARIES", - "notes": "CMS state buy-in Medicare beneficiaries minimum", - "signature": [ - "person_count", - "national", - "", - 2024, - [ - [ - "state_buy_in_medicare", - "==", - "1" - ] - ] - ] - }, - { - "variable": "person_count", - "geo_level": "national", - "geographic_id": null, - "period": 2025, - "constraints": [ - [ - "aca_ptc", - ">", - "0" - ] - ], - "value": 22380137.0, - "is_count": true, - "storage_tier": "python", - "source_path": "utils/takeup.py", - "source_row": "ACA_POST_CALIBRATION_PERSON_TARGETS", - "notes": "CMS Marketplace OEP \u2014 APTC consumers (post-calibration fallback)", - "signature": [ - "person_count", - "national", - "", - 2025, - [ - [ - "aca_ptc", - ">", - "0" - ] - ] - ] - } - ] - } - ], - "parsers_covered": [ - "aca_marketplace_state_metal_selection_2024.csv", - "aca_ptc_state.csv", - "aca_spending_and_enrollment_2024.csv", - "aca_spending_and_enrollment_2025.csv", - "aca_spending_and_enrollment_2026.csv", - "acs_housing_costs_2024.csv", - "age_state.csv", - "agi_state.csv", - "eitc_by_agi_and_children.csv", - "eitc_claim_controls.csv", - "eitc_state.csv", - "healthcare_spending.csv", - "medicaid_enrollment_2024.csv", - "medicaid_enrollment_2025.csv", - "medicaid_enrollment_2026.csv", - "np2023_d5_mid.csv", - "population_by_state.csv", - "real_estate_taxes_by_state_acs.csv", - "snap_state.csv", - "soi_targets.csv", - "spm_threshold_agi.csv" - ], - "parsers_missing": [] -} \ No newline at end of file diff --git a/backend/models.py b/backend/models.py deleted file mode 100644 index eb3a578..0000000 --- a/backend/models.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Pydantic request/response schemas.""" - -from pydantic import BaseModel - - -class TargetRow(BaseModel): - target_idx: int - target_id: int | None = None - variable: str - geo_level: str | None = None - geographic_id: str | None = None - geo_display_name: str | None = None - constraints: list[str] = [] - target_value: float - estimate: float | None = None - rel_error: float | None = None - abs_error: float | None = None - loss_contribution: float | None = None - included: bool = True - # Provenance from policy_data.db (joined at load time in pkl mode, - # read directly in dataset mode). - source: str | None = None - period: int | None = None - tolerance: float | None = None - notes: str | None = None - - # When the caller passes ?compare_run=, the joined run's - # per-row stats land here. delta = abs_rel_error_b - abs_rel_error_a - # (negative ⇒ B is closer to the target). - estimate_b: float | None = None - rel_error_b: float | None = None - abs_rel_error_b: float | None = None - delta: float | None = None - - -class TargetListResponse(BaseModel): - items: list[TargetRow] - total: int - # Populated when the request filtered to exactly one bundle that - # exists for the run and we re-evaluated estimates against that - # bundle's own h5 instead of the federal load. None means estimates - # came from the standard federal-fit pass. - bundle_evaluated: str | None = None - offset: int - limit: int - - -class ErrorDecomposition(BaseModel): - target_name: str - target_value: float - raw_sum: float - initial_estimate: float - final_estimate: float - diagnosis: str - concentration: dict - - -class ProvenanceResponse(BaseModel): - target_id: int | None = None - variable: str - value: float | None = None - period: int | None = None - source: str | None = None - tolerance: float | None = None - notes: str | None = None - active: bool | None = None - stratum_id: int | None = None - constraints: list[dict] = [] - geo_level: str | None = None - geographic_id: str | None = None - uprating_factor: float | None = None - uprated_value: float | None = None - - -class EligibilityAuditResponse(BaseModel): - target_name: str - total_contributors: int - meet_criterion: int - fail_criterion: int - pct_failing: float - weighted_contribution_from_failing: float - pct_estimate_from_failing: float - diagnosis: str - - -class ConstraintCheck(BaseModel): - variable: str - operation: str - value: str - contributors_satisfying: int - contributors_violating: int - pct_violating: float - status: str - - -class ConstraintDiffResponse(BaseModel): - target_name: str - stratum_id: int - constraints: list[ConstraintCheck] - - -class ContributorRow(BaseModel): - household_idx: int - raw_value: float - weighted_value: float - income: float | None = None - g_weight: float | None = None - in_poverty: bool | None = None - state: int | None = None - - -class HouseholdRow(BaseModel): - household_idx: int - income: float - spm_threshold: float - in_poverty: bool - initial_weight: float - final_weight: float - g_weight: float - state: int - income_decile: int - filter_variable_value: float | None = None - - -class HouseholdProfile(BaseModel): - household_idx: int - initial_weight: float - final_weight: float - g_weight: float - in_poverty: bool - state: int - cd_geoid: int - variables: dict[str, float] - - -class AttributionRow(BaseModel): - target_idx: int - target_name: str - variable: str | None = None - geo_level: str | None = None - raw_value: float - weighted_value: float - target_rel_error: float - - -class WeightSlice(BaseModel): - label: str - n: int - kish_effective_n: float - mean: float - median: float - - -class WeightDistribution(BaseModel): - kish_effective_n: float - cv: float - design_effect: float - mean: float - median: float - p5: float - p25: float - p75: float - p95: float - max: float - top_1pct_weight_share: float - top_5pct_weight_share: float - slices: list[WeightSlice] = [] - - -class HistogramBin(BaseModel): - bin_min: float - bin_max: float - count: int - - -class PovertyRateResponse(BaseModel): - spm_poverty_rate: float - spm_poverty_rate_initial_weights: float - n_poor_weighted: float - n_total_weighted_households: float - n_total_weighted_individuals: float - benchmark_census: float - - -class IncomeQuantiles(BaseModel): - p5: float - p10: float - p25: float - p50: float - p75: float - p90: float - p95: float - - -class IncomeDistributionResponse(BaseModel): - initial_weights: IncomeQuantiles - final_weights: IncomeQuantiles - - -class DecomposeRequest(BaseModel): - variable: str - subgroup: str | None = None - - -class DecomposeComponent(BaseModel): - variable: str - initial_total: float - final_total: float - shift_pct: float - - -class DecomposeResponse(BaseModel): - components: list[DecomposeComponent] - composite_initial: float | None = None - composite_final: float | None = None - - -class ConvergencePoint(BaseModel): - epoch: int - estimate: float - target: float - rel_error: float - loss: float - - -class EpochSummaryRow(BaseModel): - group: str - epoch: int - mean_abs_rel_error: float - - -class StratumDetail(BaseModel): - stratum_id: int - parent_stratum_id: int | None = None - notes: str | None = None - constraints: list[dict] = [] - children: list[dict] = [] - targets: list[dict] = [] diff --git a/backend/routes/__init__.py b/backend/routes/__init__.py deleted file mode 100644 index f36ec2f..0000000 --- a/backend/routes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Route modules.""" diff --git a/backend/routes/analysis.py b/backend/routes/analysis.py deleted file mode 100644 index 6e37b73..0000000 --- a/backend/routes/analysis.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Analyst readiness endpoints.""" - -from __future__ import annotations - -from fastapi import APIRouter, Depends, HTTPException, Query - -from backend.services.analysis_readiness import ( - audit_target_config, - build_dependency_trace, - build_domain_breakdown, - build_bundle_health, - build_readiness, - list_case_studies, - list_policyengine_variables, -) -from backend.state import AppState, get_state - -router = APIRouter() - - -@router.get("/case-studies") -def case_studies() -> dict: - return {"items": list_case_studies()} - - -@router.get("/variables") -def variables( - search: str | None = None, - limit: int = Query(100, ge=1, le=500), - state: AppState = Depends(get_state), -) -> dict: - return { - "items": list_policyengine_variables( - state, - search=search, - limit=limit, - ) - } - - -@router.get("/readiness/{case_study_id}") -def readiness( - case_study_id: str, - state: AppState = Depends(get_state), -) -> dict: - try: - return build_readiness(case_study_id, state) - except KeyError as exc: - raise HTTPException( - status_code=404, - detail=f"Unknown case study: {case_study_id}", - ) from exc - - -@router.get("/dependency/{variable}") -def dependency_trace( - variable: str, - period: int | None = None, - max_nodes: int = Query(250, ge=25, le=1000), - state: AppState = Depends(get_state), -) -> dict: - try: - return build_dependency_trace( - variable, - state, - period=period, - max_nodes=max_nodes, - ) - except KeyError as exc: - raise HTTPException( - status_code=404, - detail=f"Unknown PolicyEngine variable: {variable}", - ) from exc - except RuntimeError as exc: - raise HTTPException(status_code=503, detail=str(exc)) from exc - - -@router.get("/target-config-audit") -def target_config_audit( - variable: str | None = None, - state: AppState = Depends(get_state), -) -> dict: - return audit_target_config(state, variable=variable) - - -@router.get("/domain-breakdown") -def domain_breakdown( - variable: str | None = None, - domain_variable: str = "adjusted_gross_income", - geo_level: str | None = None, - state: AppState = Depends(get_state), -) -> dict: - return build_domain_breakdown( - state, - variable=variable, - domain_variable=domain_variable, - geo_level=geo_level, - ) - - -@router.get("/bundle-health") -def bundle_health( - dataset_file: str, - limit: int = Query(10, ge=1, le=100), - state: AppState = Depends(get_state), -) -> dict: - try: - return build_bundle_health( - state, - dataset_file=dataset_file, - limit=limit, - ) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) from exc diff --git a/backend/routes/compare.py b/backend/routes/compare.py deleted file mode 100644 index fa35d5f..0000000 --- a/backend/routes/compare.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Pairwise run comparison. - -Same dataset, two runs. Joins both runs' ``targets_enriched`` on -``target_id`` and surfaces: - -- Headline deltas: how much closer (or farther) each run is overall. -- Biggest movers: per-target rows where the absolute relative error - shifted most between A and B, split into improvements and regressions. - -We do NOT use the standard ``get_state`` dependency because we need two -states at once. The route resolves both runs through the registry -directly. -""" - -from __future__ import annotations - -from typing import Annotated - -import numpy as np -import pandas as pd -from fastapi import APIRouter, HTTPException, Query, Request - -router = APIRouter() - - -_DEFAULT_MOVERS = 25 - - -def _headline(df: pd.DataFrame) -> dict: - """Headline error stats over the included subset of a run.""" - if df.empty or "abs_rel_error" not in df.columns: - return _empty_headline() - included = df[df["included"].astype(bool)] if "included" in df.columns else df - abs_err = included["abs_rel_error"].to_numpy() - finite = abs_err[np.isfinite(abs_err)] - if not len(finite): - return _empty_headline(n_total=int(len(df)), n_included=int(len(included))) - return { - "n_total": int(len(df)), - "n_included": int(len(included)), - "n_with_estimate": int(np.sum(~np.isnan(abs_err))), - "median_abs_rel_error": float(np.median(finite)), - "mean_abs_rel_error": float(np.mean(finite)), - "p95_abs_rel_error": float(np.percentile(finite, 95)), - "pct_within_5pct": float(np.mean(finite <= 0.05)), - "pct_within_10pct": float(np.mean(finite <= 0.10)), - "pct_within_25pct": float(np.mean(finite <= 0.25)), - } - - -def _empty_headline(n_total: int = 0, n_included: int = 0) -> dict: - return { - "n_total": n_total, - "n_included": n_included, - "n_with_estimate": 0, - "median_abs_rel_error": None, - "mean_abs_rel_error": None, - "p95_abs_rel_error": None, - "pct_within_5pct": None, - "pct_within_10pct": None, - "pct_within_25pct": None, - } - - -def _movers(merged: pd.DataFrame, top_n: int) -> dict: - """Return ``top_n`` rows in each direction (improvements / regressions). - - Rank by signed change in absolute relative error: - delta = abs_rel_error_b - abs_rel_error_a - Negative ⇒ B is closer (improvement). Positive ⇒ B is farther - (regression). Rows missing an estimate in either run are excluded. - """ - valid = merged.dropna(subset=["abs_rel_error_a", "abs_rel_error_b"]) - if valid.empty: - return {"improved": [], "regressed": []} - delta = valid["abs_rel_error_b"] - valid["abs_rel_error_a"] - ranked = valid.assign(delta=delta) - - cols = [ - "target_id", "variable", "geo_level", "geographic_id", - "value", "estimate_a", "estimate_b", - "rel_error_a", "rel_error_b", - "abs_rel_error_a", "abs_rel_error_b", "delta", - ] - improved = ranked.nsmallest(top_n, "delta")[cols].to_dict(orient="records") - regressed = ranked.nlargest(top_n, "delta")[cols].to_dict(orient="records") - return {"improved": improved, "regressed": regressed} - - -def _by_variable(merged: pd.DataFrame, top_n: int) -> list[dict]: - """Per-variable rollup: mean abs_rel_error for each run + counts of - targets that improved / regressed by >1pp.""" - valid = merged.dropna(subset=["abs_rel_error_a", "abs_rel_error_b"]) - if valid.empty: - return [] - grouped = valid.groupby("variable").agg( - n_targets=("target_id", "size"), - mean_a=("abs_rel_error_a", "mean"), - mean_b=("abs_rel_error_b", "mean"), - ) - delta = valid["abs_rel_error_b"] - valid["abs_rel_error_a"] - improvement_counts = valid.assign(d=delta).groupby("variable")["d"].apply( - lambda s: int((s < -0.01).sum()) - ) - regression_counts = valid.assign(d=delta).groupby("variable")["d"].apply( - lambda s: int((s > 0.01).sum()) - ) - grouped["n_improved"] = improvement_counts - grouped["n_regressed"] = regression_counts - grouped["mean_delta"] = grouped["mean_b"] - grouped["mean_a"] - grouped = grouped.sort_values("mean_delta", ascending=False) - rows = [] - for var, r in grouped.head(top_n).iterrows(): - rows.append({ - "variable": str(var), - "n_targets": int(r["n_targets"]), - "mean_abs_rel_error_a": float(r["mean_a"]), - "mean_abs_rel_error_b": float(r["mean_b"]), - "mean_delta": float(r["mean_delta"]), - "n_improved": int(r["n_improved"]), - "n_regressed": int(r["n_regressed"]), - }) - return rows - - -def _scrub(obj): - """Replace non-finite floats with None for JSON serialization.""" - if isinstance(obj, dict): - return {k: _scrub(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_scrub(x) for x in obj] - if isinstance(obj, float) and not np.isfinite(obj): - return None - return obj - - -@router.get("/compare") -def compare( - request: Request, - dataset: Annotated[str, Query(description="Dataset id, e.g. us-data")], - run_a: Annotated[str, Query(description="Baseline run id")], - run_b: Annotated[str, Query(description="Comparison run id")], - top_n: Annotated[int, Query(ge=5, le=200)] = _DEFAULT_MOVERS, -) -> dict: - """Side-by-side comparison of two runs of the same dataset. - - Joins on ``target_id`` so the comparison is target-equivalent (both - runs must come from the same dataset → same targets table). Same run - on both sides is allowed; deltas will be all zero. - """ - if not run_a or not run_b: - raise HTTPException(status_code=400, detail="run_a and run_b are required") - - registry = request.app.state.registry - try: - state_a = registry.get(dataset, run_a) - state_b = registry.get(dataset, run_b) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) - except Exception as exc: - raise HTTPException( - status_code=500, - detail=f"Failed to load one of the runs: {exc}", - ) - - df_a = state_a.targets_enriched - df_b = state_b.targets_enriched - if df_a.empty or df_b.empty: - raise HTTPException( - status_code=400, - detail="One of the runs has no targets loaded.", - ) - - headline_a = _headline(df_a) - headline_b = _headline(df_b) - - # Join on target_id. Carry along identity columns (value etc.) - # from A; both runs should agree on these since dataset is shared. - keep = ["target_id", "variable", "geo_level", "geographic_id", "value"] - a = df_a[keep + ["estimate", "rel_error", "abs_rel_error"]].rename( - columns={ - "estimate": "estimate_a", - "rel_error": "rel_error_a", - "abs_rel_error": "abs_rel_error_a", - } - ) - b = df_b[["target_id", "estimate", "rel_error", "abs_rel_error"]].rename( - columns={ - "estimate": "estimate_b", - "rel_error": "rel_error_b", - "abs_rel_error": "abs_rel_error_b", - } - ) - merged = a.merge(b, on="target_id", how="inner") - - payload = { - "dataset": dataset, - "run_a": run_a, - "run_b": run_b, - "headline_a": headline_a, - "headline_b": headline_b, - "movers": _movers(merged, top_n), - "by_variable": _by_variable(merged, top_n), - } - return _scrub(payload) diff --git a/backend/routes/geography.py b/backend/routes/geography.py deleted file mode 100644 index eb1bb12..0000000 --- a/backend/routes/geography.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Geography listing and lookup endpoints.""" - -import numpy as np -from fastapi import APIRouter, Depends - -from backend.services.geo_utils import ( - cd_display_name, - list_states, - state_name, -) -from backend.state import AppState, get_state - -router = APIRouter() - - -@router.get("/states") -def get_states() -> list[dict]: - """List all states with FIPS, name, and abbreviation.""" - return list_states() - - -@router.get("/districts") -def get_districts( - state: AppState = Depends(get_state), -) -> list[dict]: - """List all congressional districts present in the dataset.""" - cd_col = state.households_df["cd_geoid"].values - unique_cds = sorted(set(int(x) for x in cd_col if x != 0)) - return [ - {"cd_geoid": cd, "name": cd_display_name(cd)} - for cd in unique_cds - ] - - -@router.get("/districts/{state_fips}") -def get_districts_for_state( - state_fips: int, - state: AppState = Depends(get_state), -) -> list[dict]: - """List congressional districts in a specific state.""" - cd_col = state.households_df["cd_geoid"].values - unique_cds = sorted(set( - int(x) for x in cd_col - if x != 0 and int(x) // 100 == state_fips - )) - return [ - {"cd_geoid": cd, "name": cd_display_name(cd)} - for cd in unique_cds - ] diff --git a/backend/routes/microplex.py b/backend/routes/microplex.py deleted file mode 100644 index 962c7c8..0000000 --- a/backend/routes/microplex.py +++ /dev/null @@ -1,2086 +0,0 @@ -"""Microplex target-performance view. - -Reads the parity / regression / drilldown JSONs that microplex-us -commits under ``artifacts/`` directly from GitHub. Newer Microplex runs -write full per-target diagnostics as a run-bundle artifact named -``policyengine_native_target_diagnostics`` with path -``pe_native_target_diagnostics.json``. Those run bundles, the run index, -native audit, and output H5 are generated artifacts, not committed public -JSONs, so the committed summaries are the only public signal we can pull -in without credentials or a separately supplied artifact root. - -This view is intentionally read-only and aggregate. When microplex -publishes its generated target diagnostics or H5 artifacts, this route can -serve Microplex's standalone target-oracle performance table and optionally -compare it with us-data. -""" - -from __future__ import annotations - -import json -import logging -import os -import time -from functools import cmp_to_key, lru_cache -from pathlib import Path -from typing import Any - -from fastapi import APIRouter, HTTPException -import numpy as np - -logger = logging.getLogger(__name__) -router = APIRouter() - - -# Pinned filenames in the microplex-us repo. If the team renames or -# rotates these the dashboard will surface a 404 with the path so we can -# bump the constants. -_PARITY_PATH = ( - "artifacts/live_pe_native_cps_puf_rich_broad_fixed_20260329/" - "20260329T175330Z-057066af/pe_us_data_rebuild_parity.json" -) -_REGRESSION_SUMMARY_PATH = ( - "artifacts/live_pe_us_data_rebuild_checkpoint_modelpass_" - "regression_summary_20260410.json" -) -_IRS_DRILLDOWN_PATH = ( - "artifacts/live_pe_us_data_rebuild_checkpoint_national_irs_" - "other_drilldown_20260410.json" -) -_RUN_LEVEL_TARGET_DIAGNOSTICS_PATH = "pe_native_target_diagnostics.json" -_RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY = ( - "policyengine_native_target_diagnostics" -) -_LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH = ( - "artifacts/pe_native_target_diagnostics_current.json" -) - -_GITHUB_RAW = "https://raw.githubusercontent.com/PolicyEngine/microplex-us/main" - -_ARTIFACTS = { - "parity": _PARITY_PATH, - "regression_summary": _REGRESSION_SUMMARY_PATH, - "irs_drilldown": _IRS_DRILLDOWN_PATH, -} - -_GENERATED_ARTIFACT_CONTRACT = [ - { - "name": "full_target_diagnostics", - "path_hint": _RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - "manifest_key": _RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - "legacy_static_dashboard_path": _LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH, - "producer": "build_us_pe_native_target_diagnostics_payload", - "public_committed": False, - "description": ( - "Full per-target PE-native rows saved inside each newer Microplex " - "run bundle. The rows show Microplex aggregate estimates against " - "target values, with us-data comparator fields when present." - ), - }, - { - "name": "dashboard_payload", - "path_hint": "artifacts/microplex_dashboard_current.json", - "producer": "microplex-us-dashboard", - "public_committed": False, - "description": "Living dashboard payload with score runs, logs, and target diagnostics.", - }, - { - "name": "native_scores", - "path_hint": "policyengine_native_scores.json", - "producer": "compute_us_pe_native_scores", - "public_committed": False, - "description": "Compact broad native-loss summary for one artifact bundle.", - }, - { - "name": "native_audit", - "path_hint": "pe_us_data_rebuild_native_audit.json", - "producer": "build_policyengine_us_data_rebuild_native_audit", - "public_committed": False, - "description": "Top family and target regressions plus support audit evidence.", - }, - { - "name": "run_index", - "path_hint": "run_index.duckdb", - "producer": "append_us_microplex_run_index_entry", - "public_committed": False, - "description": "DuckDB index for querying target deltas across saved runs.", - }, -] - -_TARGET_DIAGNOSTIC_ROW_FIELDS = [ - "target_id", - "family", - "in_loss", - "supported_by_microplex", - "baseline_dataset", - "candidate_dataset", - "baseline_label", - "candidate_label", - "target_value", - "us_data_aggregate", - "microplex_aggregate", - "us_data_absolute_error", - "microplex_absolute_error", - "us_data_relative_error", - "microplex_relative_error", - "delta_absolute_error", - "delta_relative_error", - "loss_contribution", -] - -# In-process cache: the JSONs change only when microplex-us commits, so a -# few minutes of TTL is plenty. -_CACHE: dict[str, tuple[float, Any]] = {} -_TTL_SECONDS = 300 -_REFORM_COMPARISON_CACHE: dict[ - tuple[str, str, str, str, int], - tuple[float, dict[str, Any]], -] = {} -_REFORM_COMPARISON_TTL_SECONDS = 900 -_BUDGET_BENCHMARK_CACHE: dict[ - tuple[str, str, int, int], - tuple[float, dict[str, Any]], -] = {} -_BUDGET_BENCHMARK_TTL_SECONDS = 900 - -_REFORM_PRESETS: dict[str, dict[str, Any]] = { - "american_family_act_2025": { - "id": "american_family_act_2025", - "label": "American Family Act 2025 CTC expansion", - "description": ( - "Expands the Child Tax Credit using the American Family Act " - "structure implemented in PolicyEngine-US." - ), - "variable": "ctc", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://www.policyengine.org/us/research/american-family-act-2025", - }, - "working_parents_tax_relief_act_2026": { - "id": "working_parents_tax_relief_act_2026", - "label": "Working Parents Tax Relief Act EITC enhancement", - "description": ( - "Enhances the EITC for parents with young children, using the " - "2026 PolicyEngine-US reform implementation." - ), - "variable": "eitc", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://www.policyengine.org/us/working-parents-tax-relief-act", - }, - "halve_joint_eitc_phase_out_rate": { - "id": "halve_joint_eitc_phase_out_rate", - "label": "Halve joint-filer EITC phase-out rate", - "description": ( - "Structural PolicyEngine-US reform that halves the EITC phase-out " - "rate for joint filers. Useful as a simple smoke test." - ), - "variable": "eitc", - "entity": "tax_unit", - "period": 2024, - "unit": "USD", - "source_url": None, - }, - "wyden_smith_ctc_2024": { - "id": "wyden_smith_ctc_2024", - "label": "Wyden-Smith / TRAFWA CTC provisions", - "description": ( - "Implements the 2024 Child Tax Credit provisions from H.R. 7024: " - "per-child ACTC phase-in, prior-year earnings lookback, 2024 ACTC " - "refundable maximum increase, and 2024 CTC indexing." - ), - "variable": "ctc", - "entity": "tax_unit", - "period": 2024, - "unit": "USD", - "source_url": "https://www.policyengine.org/us/research/trafwa-ctc", - }, - "kypa_ctc_2026": { - "id": "kypa_ctc_2026", - "label": "Keep Your Pay Act expanded CTC", - "description": ( - "Uses the PolicyEngine-US American Family Act-style CTC " - "implementation to approximate KYPA's expanded, fully refundable " - "CTC in 2026." - ), - "variable": "ctc", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - }, - "kypa_childless_eitc_2026": { - "id": "kypa_childless_eitc_2026", - "label": "Keep Your Pay Act childless EITC expansion", - "description": ( - "Applies KYPA's childless-worker EITC parameters: lower minimum " - "age, no maximum age, doubled phase-in and phase-out rates, and " - "a higher maximum credit." - ), - "variable": "eitc", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - }, - "obbba_no_tax_on_tips_repeal_2026": { - "id": "obbba_no_tax_on_tips_repeal_2026", - "label": "OBBBA no tax on tips provision", - "description": ( - "Counterfactual removal of the current-law tip income deduction. " - "The tax increase from removal is interpreted as the standalone " - "modeled cost of keeping the provision." - ), - "variable": "income_tax", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://www.jct.gov/publications/2025/jcx-35-25/", - }, - "obbba_no_tax_on_overtime_repeal_2026": { - "id": "obbba_no_tax_on_overtime_repeal_2026", - "label": "OBBBA no tax on overtime provision", - "description": ( - "Counterfactual removal of the current-law overtime income " - "deduction. The tax increase from removal is interpreted as the " - "standalone modeled cost of keeping the provision." - ), - "variable": "income_tax", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://www.jct.gov/publications/2025/jcx-35-25/", - }, - "obbba_senior_deduction_repeal_2026": { - "id": "obbba_senior_deduction_repeal_2026", - "label": "OBBBA senior deduction provision", - "description": ( - "Counterfactual removal of the current-law additional senior " - "deduction. The tax increase from removal is interpreted as the " - "standalone modeled cost of keeping the provision." - ), - "variable": "income_tax", - "entity": "tax_unit", - "period": 2026, - "unit": "USD", - "source_url": "https://www.jct.gov/publications/2025/jcx-26-25r/", - }, -} - -_BUDGET_BENCHMARKS: list[dict[str, Any]] = [ - { - "id": "american_family_act_2025", - "title": "American Family Act 2025 CTC expansion", - "policy_area": "Child Tax Credit", - "live_reform_id": None, - "budget_effect_rule": "credit_delta_is_cost", - "benchmark_period": "2026 annual", - "comparison_status": "model_context_no_third_party_score", - "external_estimates": [ - { - "source": "CBO/JCT", - "source_type": "official_score", - "url": "https://www.congress.gov/bill/119th-congress/house-bill/2763", - "estimate": None, - "estimate_label": "No public CBO/JCT score found for H.R.2763 / S.1393.", - "period": "not available", - } - ], - "notes": ( - "No independent public budget score is attached for this bill. " - "PolicyEngine has a published static analysis, but that is not a " - "third-party benchmark and is intentionally excluded from the " - "external comparison slot." - ), - }, - { - "id": "working_parents_tax_relief_act_2026", - "title": "Working Parents Tax Relief Act EITC enhancement", - "policy_area": "Earned Income Tax Credit", - "live_reform_id": None, - "budget_effect_rule": "credit_delta_is_cost", - "benchmark_period": "2026 annual", - "comparison_status": "model_context_partial_external_context", - "external_estimates": [ - { - "source": "Thomson Reuters coverage", - "source_type": "third_party_context", - "url": "https://tax.thomsonreuters.com/news/bill-seeks-earned-income-tax-credit-boost-per-child-for-working-parents/", - "estimate": None, - "estimate_label": ( - "Third-party coverage found; no single budget score is " - "attached in this catalog." - ), - "period": "not available", - }, - { - "source": "PolicyEngine policy page", - "source_type": "published_model_result", - "url": "https://www.policyengine.org/us/working-parents-tax-relief-act", - "estimate": None, - "estimate_label": "PolicyEngine analysis; not a CBO/JCT comparator.", - "period": "2026+", - }, - ], - "notes": ( - "Live values are annual aggregate EITC deltas for the implemented " - "PolicyEngine-US reform." - ), - }, - { - "id": "wyden_smith_ctc_2024", - "title": "Wyden-Smith / TRAFWA CTC provisions", - "policy_area": "Child Tax Credit", - "live_reform_id": "wyden_smith_ctc_2024", - "budget_effect_rule": "credit_delta_is_cost", - "benchmark_period": "2024 annual", - "comparison_status": "live_model_with_third_party_score", - "external_estimates": [ - { - "source": "Joint Committee on Taxation", - "source_type": "jct", - "url": "https://waysandmeans.house.gov/wp-content/uploads/2024/01/Estimated-Revenue-Effects-of-H.R.-7024.pdf", - "estimate": 10_700_000_000, - "estimate_label": ( - "$10.7B 2024 cost for the combined CTC provisions, as " - "reported in PolicyEngine's JCT comparison table." - ), - "period": "2024", - "comparable_to_live_annual_result": True, - }, - { - "source": "Joint Committee on Taxation", - "source_type": "jct", - "url": "https://waysandmeans.house.gov/wp-content/uploads/2024/01/Estimated-Revenue-Effects-of-H.R.-7024.pdf", - "estimate": 33_493_000_000, - "estimate_label": ( - "$33.493B 2024-2033 revenue effect for the Tax Relief for " - "Working Families line in JCX-3-24." - ), - "period": "2024-2033", - "comparable_to_live_annual_result": False, - }, - ], - "notes": ( - "This row is the first true third-party benchmark: the live model " - "runs the CTC outcome against a JCT provision-level score. The " - "annual comparison uses PolicyEngine's published JCT comparison " - "table for the 2024 CTC provisions; the decade score is included " - "as context but is not directly comparable to the single-year live " - "run." - ), - }, - { - "id": "kypa_ctc_2026", - "title": "Keep Your Pay Act expanded CTC", - "policy_area": "Child Tax Credit", - "live_reform_id": "kypa_ctc_2026", - "budget_effect_rule": "credit_delta_is_cost", - "benchmark_period": "2026 annual", - "comparison_status": "live_model_with_third_party_score", - "external_estimates": [ - { - "source": "Penn Wharton Budget Model", - "source_type": "pwbm", - "url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - "estimate": 140_500_000_000, - "estimate_label": ( - "$140.5B FY2027 revenue loss for KYPA's expanded Child " - "Tax Credit provision. This is the closest full-year " - "fiscal proxy for a calendar-year 2026 microsim." - ), - "period": "FY2027 proxy for TY2026", - "comparable_to_live_annual_result": True, - }, - { - "source": "Penn Wharton Budget Model", - "source_type": "pwbm", - "url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - "estimate": 2_500_000_000, - "estimate_label": ( - "$2.5B FY2026 revenue loss. Included as timing context; " - "not comparable to a full calendar-year 2026 tax microsim." - ), - "period": "FY2026 timing context", - "comparable_to_live_annual_result": False, - }, - { - "source": "Penn Wharton Budget Model", - "source_type": "pwbm", - "url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - "estimate": 1_261_600_000_000, - "estimate_label": ( - "$1.2616T FY2026-2035 revenue loss for KYPA's expanded " - "Child Tax Credit provision." - ), - "period": "FY2026-2035", - "comparable_to_live_annual_result": False, - }, - ], - "notes": ( - "PWBM provides separable KYPA CTC estimates. The live dashboard " - "comparison uses the 2026 annual CTC delta against PWBM's FY2027 " - "line as a full-year fiscal proxy. PWBM's FY2026 line is much " - "smaller because refundable credit timing shifts most TY2026 " - "cost into FY2027. The PolicyEngine-US reform is an AFA-style " - "approximation: it matches the main child amounts, refundability, " - "and phaseout structure, but its newborn-bonus formula is not " - "guaranteed to match PWBM exactly." - ), - }, - { - "id": "kypa_childless_eitc_2026", - "title": "Keep Your Pay Act childless-worker EITC", - "policy_area": "Earned Income Tax Credit", - "live_reform_id": "kypa_childless_eitc_2026", - "budget_effect_rule": "credit_delta_is_cost", - "benchmark_period": "2026 annual", - "comparison_status": "live_model_with_third_party_score", - "external_estimates": [ - { - "source": "Penn Wharton Budget Model", - "source_type": "pwbm", - "url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - "estimate": 7_200_000_000, - "estimate_label": ( - "$7.2B FY2027 revenue loss for KYPA's childless-worker " - "EITC expansion. This is the closest full-year fiscal " - "proxy for a calendar-year 2026 microsim." - ), - "period": "FY2027 proxy for TY2026", - "comparable_to_live_annual_result": True, - }, - { - "source": "Penn Wharton Budget Model", - "source_type": "pwbm", - "url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - "estimate": 800_000_000, - "estimate_label": ( - "$0.8B FY2026 revenue loss. Included as timing context; " - "not comparable to a full calendar-year 2026 tax microsim." - ), - "period": "FY2026 timing context", - "comparable_to_live_annual_result": False, - }, - { - "source": "Penn Wharton Budget Model", - "source_type": "pwbm", - "url": "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - "estimate": 63_800_000_000, - "estimate_label": ( - "$63.8B FY2026-2035 revenue loss for KYPA's childless-" - "worker EITC expansion." - ), - "period": "FY2026-2035", - "comparable_to_live_annual_result": False, - }, - ], - "notes": ( - "PWBM provides a separable childless-worker EITC estimate. The " - "live comparison applies the same 2026 parameters described by " - "PWBM and compares to the FY2027 line as a full-year fiscal " - "proxy: minimum age 19, no maximum age, 15.3% phase-in and " - "phase-out rates, about a $1,502 maximum credit, and about an " - "$11,610 phase-out start." - ), - }, - { - "id": "obbba_no_tax_on_tips_repeal_2026", - "title": "OBBBA no tax on tips", - "policy_area": "Federal individual income tax", - "live_reform_id": "obbba_no_tax_on_tips_repeal_2026", - "budget_effect_rule": "repeal_tax_delta_is_provision_cost", - "benchmark_period": "2026 annual", - "comparison_status": "waterfall_external_score_context", - "external_estimates": [ - { - "source": "Joint Committee on Taxation JCX-35-25", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-35-25/", - "estimate": 10_121_000_000, - "estimate_label": ( - "$10.121B FY2026 revenue loss for the no-tax-on-tips " - "line item in JCT's OBBBA table. Treat as package-order " - "context, not a standalone oracle." - ), - "period": "FY2026", - "comparable_to_live_annual_result": False, - }, - { - "source": "Joint Committee on Taxation JCX-35-25", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-35-25/", - "estimate": 31_664_000_000, - "estimate_label": ( - "$31.664B 2025-2034 revenue loss for the no-tax-on-tips " - "line item in JCT's OBBBA table. Treat as package-order " - "context, not a standalone oracle." - ), - "period": "2025-2034", - }, - ], - "notes": ( - "This is modeled as a one-provision counterfactual because the " - "installed PolicyEngine-US baseline already includes the OBBBA " - "tip deduction. The dashboard removes only that deduction and " - "uses the resulting income-tax increase as the provision cost. " - "JCT's OBBBA table is not a clean standalone target for this " - "counterfactual because it uses a specified OBBBA baseline and " - "package-order interactions. The live model is calendar-year 2026; " - "the JCT annual entry is fiscal-year 2026." - ), - }, - { - "id": "obbba_no_tax_on_overtime_repeal_2026", - "title": "OBBBA no tax on overtime", - "policy_area": "Federal individual income tax", - "live_reform_id": "obbba_no_tax_on_overtime_repeal_2026", - "budget_effect_rule": "repeal_tax_delta_is_provision_cost", - "benchmark_period": "2026 annual", - "comparison_status": "waterfall_external_score_context", - "external_estimates": [ - { - "source": "Joint Committee on Taxation JCX-35-25", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-35-25/", - "estimate": 32_806_000_000, - "estimate_label": ( - "$32.806B FY2026 revenue loss for the no-tax-on-overtime " - "line item in JCT's OBBBA table. Treat as package-order " - "context, not a standalone oracle." - ), - "period": "FY2026", - "comparable_to_live_annual_result": False, - }, - { - "source": "Joint Committee on Taxation JCX-35-25", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-35-25/", - "estimate": 89_573_000_000, - "estimate_label": ( - "$89.573B 2025-2034 revenue loss for the no-tax-on-" - "overtime line item in JCT's OBBBA table. Treat as " - "package-order context, not a standalone oracle." - ), - "period": "2025-2034", - }, - ], - "notes": ( - "This is modeled as a one-provision counterfactual because the " - "installed PolicyEngine-US baseline already includes the OBBBA " - "overtime deduction. The dashboard removes only that deduction " - "and uses the resulting income-tax increase as the provision cost. " - "JCT's OBBBA table is not a clean standalone target for this " - "counterfactual because it uses a specified OBBBA baseline and " - "package-order interactions. The live model is calendar-year 2026; " - "the JCT annual entry is fiscal-year 2026." - ), - }, - { - "id": "obbba_senior_deduction_repeal_2026", - "title": "OBBBA senior deduction", - "policy_area": "Federal individual income tax", - "live_reform_id": "obbba_senior_deduction_repeal_2026", - "budget_effect_rule": "repeal_tax_delta_is_provision_cost", - "benchmark_period": "2026 annual", - "comparison_status": "waterfall_external_score_context", - "external_estimates": [ - { - "source": "Joint Committee on Taxation JCX-26-25R", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-26-25r/", - "estimate": 16_464_000_000, - "estimate_label": ( - "$16.464B FY2026 revenue loss for the enhanced senior " - "deduction line item in JCT's OBBBA table. Treat as " - "package-order context, not a standalone oracle." - ), - "period": "FY2026", - "comparable_to_live_annual_result": False, - }, - { - "source": "Joint Committee on Taxation JCX-26-25R", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-26-25r/", - "estimate": 66_263_000_000, - "estimate_label": ( - "$66.263B 2025-2034 revenue loss for the enhanced senior " - "deduction line item in JCT's OBBBA table. Treat as " - "package-order context, not a standalone oracle." - ), - "period": "2025-2034", - }, - ], - "notes": ( - "This is modeled as a one-provision counterfactual because the " - "installed PolicyEngine-US baseline already includes the OBBBA " - "senior deduction. The dashboard removes only that deduction and " - "uses the resulting income-tax increase as the provision cost. " - "JCT's OBBBA table is not a clean standalone target for this " - "counterfactual because it uses a specified OBBBA baseline and " - "package-order interactions. The live model is calendar-year 2026; " - "the JCT annual entry is fiscal-year 2026." - ), - }, - { - "id": "tcja_extension_2026_2035", - "title": "TCJA individual provisions extension", - "policy_area": "Federal individual income tax", - "live_reform_id": None, - "budget_effect_rule": "full_budget_score", - "benchmark_period": "2026-2035", - "comparison_status": "external_score_available_reform_not_wired", - "external_estimates": [ - { - "source": "CBO/JCT", - "source_type": "cbo_jct", - "url": "https://www.policyengine.org/us/research/tcja-extension", - "estimate": 3_877_600_000_000, - "estimate_label": "$3.8776T cost over 2026-2035", - "period": "2026-2035", - }, - { - "source": "CRFB", - "source_type": "third_party_score", - "url": "https://www.policyengine.org/us/research/tcja-extension", - "estimate": 3_830_000_000_000, - "estimate_label": "$3.83T cost over 2026-2035", - "period": "2026-2035", - }, - { - "source": "PolicyEngine dynamic", - "source_type": "published_model_result", - "url": "https://www.policyengine.org/us/research/tcja-extension", - "estimate": 3_885_500_000_000, - "estimate_label": "$3.8855T cost over 2026-2035", - "period": "2026-2035", - }, - ], - "notes": ( - "External benchmark is strong, but the dashboard does not yet have " - "a matching live TCJA-extension reform preset for us-data and " - "Microplex." - ), - }, - { - "id": "final_2025_reconciliation_tax", - "title": "Final 2025 reconciliation individual income tax provisions", - "policy_area": "Federal individual income tax", - "live_reform_id": None, - "budget_effect_rule": "full_budget_score", - "benchmark_period": "2026-2035", - "comparison_status": "external_score_available_reform_not_wired", - "external_estimates": [ - { - "source": "PolicyEngine static analysis", - "source_type": "published_model_result", - "url": "https://www.policyengine.org/us/research/final-2025-reconciliation-tax", - "estimate": 3_785_000_000_000, - "estimate_label": "$3.785T cost over 2026-2035", - "period": "2026-2035", - }, - { - "source": "JCT JCX-26-25", - "source_type": "jct", - "url": "https://www.jct.gov/publications/2025/jcx-26-25/", - "estimate": None, - "estimate_label": "Official JCT revenue estimate available; row-level match not wired.", - "period": "2025 budget reconciliation", - }, - ], - "notes": ( - "PolicyEngine-US baseline now contains many OBBBA provisions, so " - "a live before/after comparison needs an explicit counterfactual " - "reform branch rather than a simple preset." - ), - }, -] - - -def _fetch_json(path: str) -> Any: - cached = _CACHE.get(path) - if cached and time.time() - cached[0] < _TTL_SECONDS: - return cached[1] - import urllib.request - url = f"{_GITHUB_RAW}/{path}" - logger.info("Fetching microplex artifact: %s", url) - try: - with urllib.request.urlopen(url, timeout=30) as resp: - body = resp.read().decode("utf-8") - data = json.loads(body) - _CACHE[path] = (time.time(), data) - return data - except Exception as exc: - raise HTTPException( - status_code=502, - detail=f"Failed to fetch microplex artifact {path}: {exc}", - ) - - -def _scrub(obj): - """Replace non-finite floats with None for JSON serialization.""" - if isinstance(obj, dict): - return {k: _scrub(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_scrub(x) for x in obj] - if isinstance(obj, float): - import math - if not math.isfinite(obj): - return None - return obj - - -def _configured_artifact_roots() -> list[Path]: - raw = os.environ.get("MICROPLEX_ARTIFACT_ROOTS") or os.environ.get( - "MICROPLEX_ARTIFACT_ROOT" - ) - if not raw: - return [] - separators = [os.pathsep, ","] - parts = [raw] - for separator in separators: - next_parts: list[str] = [] - for part in parts: - next_parts.extend(part.split(separator)) - parts = next_parts - return [Path(part).expanduser().resolve() for part in parts if part.strip()] - - -def _read_json_file(path: Path) -> Any | None: - try: - return json.loads(path.read_text()) - except (OSError, json.JSONDecodeError): - return None - - -def _resolve_artifact_path(bundle_dir: Path, value: Any) -> Path | None: - if not isinstance(value, str) or not value: - return None - path = Path(value).expanduser() - if not path.is_absolute(): - path = bundle_dir / path - return path.resolve() - - -def _discover_configured_run_bundles() -> dict[str, Any]: - roots = _configured_artifact_roots() - bundles: list[dict[str, Any]] = [] - missing_roots = [] - for root in roots: - if not root.exists(): - missing_roots.append(str(root)) - continue - for manifest_path in root.rglob("manifest.json"): - manifest = _read_json_file(manifest_path) - if not isinstance(manifest, dict): - continue - bundle_dir = manifest_path.parent - artifacts = manifest.get("artifacts") - artifacts = artifacts if isinstance(artifacts, dict) else {} - diagnostics_path = _resolve_artifact_path( - bundle_dir, - artifacts.get(_RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY), - ) - native_scores_path = _resolve_artifact_path( - bundle_dir, - artifacts.get("policyengine_native_scores") - or "policyengine_native_scores.json", - ) - native_audit_path = _resolve_artifact_path( - bundle_dir, - artifacts.get("policyengine_native_audit") - or "pe_us_data_rebuild_native_audit.json", - ) - policyengine_dataset_path = _resolve_artifact_path( - bundle_dir, - artifacts.get("policyengine_dataset") or "policyengine_us.h5", - ) - modified_at = manifest_path.stat().st_mtime - bundles.append( - { - "artifact_id": ( - manifest.get("artifact_id") - or manifest.get("artifactId") - or bundle_dir.name - ), - "artifact_dir": str(bundle_dir), - "manifest_path": str(manifest_path), - "modified_at_unix": modified_at, - "target_diagnostics_path": ( - str(diagnostics_path) if diagnostics_path is not None else None - ), - "target_diagnostics_exists": ( - diagnostics_path.exists() - if diagnostics_path is not None - else False - ), - "native_scores_path": ( - str(native_scores_path) if native_scores_path is not None else None - ), - "native_scores_exists": ( - native_scores_path.exists() - if native_scores_path is not None - else False - ), - "native_audit_path": ( - str(native_audit_path) if native_audit_path is not None else None - ), - "native_audit_exists": ( - native_audit_path.exists() - if native_audit_path is not None - else False - ), - "policyengine_dataset_path": ( - str(policyengine_dataset_path) - if policyengine_dataset_path is not None - else None - ), - "policyengine_dataset_exists": ( - policyengine_dataset_path.exists() - if policyengine_dataset_path is not None - else False - ), - } - ) - - bundles.sort(key=lambda item: item["modified_at_unix"], reverse=True) - latest = bundles[0] if bundles else None - return { - "artifact_root_env": "MICROPLEX_ARTIFACT_ROOTS", - "single_artifact_root_env": "MICROPLEX_ARTIFACT_ROOT", - "configured_artifact_roots": [str(root) for root in roots], - "missing_artifact_roots": missing_roots, - "detected_run_bundle_count": len(bundles), - "detected_target_diagnostics_count": sum( - 1 for bundle in bundles if bundle["target_diagnostics_exists"] - ), - "latest_run_bundle": latest, - "sampled_run_bundles": bundles[:10], - } - - -def _latest_microplex_policyengine_dataset() -> dict[str, Any] | None: - discovery = _discover_configured_run_bundles() - latest = discovery.get("latest_run_bundle") - if not isinstance(latest, dict): - return None - if not latest.get("policyengine_dataset_exists"): - return None - path = latest.get("policyengine_dataset_path") - if not isinstance(path, str) or not path: - return None - return latest - - -def _working_parents_tax_relief_parameter_reform(): - from policyengine_core.periods import instant - from policyengine_us.model_api import Reform - - class working_parents_tax_relief_parameter_reform(Reform): - def apply(self): - def modify_parameters(parameters): - parameter = ( - parameters.gov.contrib.congress.mcdonald_rivet - .working_parents_tax_relief_act.in_effect - ) - parameter.update( - start=instant("2026-01-01"), - stop=instant("2035-12-31"), - value=True, - ) - return parameters - - self.modify_parameters(modify_parameters) - - return working_parents_tax_relief_parameter_reform - - -def _wyden_smith_ctc_2024_parameter_reform(): - from policyengine_core.periods import instant - from policyengine_us.model_api import Reform - - class wyden_smith_ctc_2024_parameter_reform(Reform): - def apply(self): - def modify_parameters(parameters): - wyden_smith = parameters.gov.contrib.congress.wyden_smith - wyden_smith.per_child_actc_phase_in.update( - start=instant("2024-01-01"), - stop=instant("2024-12-31"), - value=True, - ) - wyden_smith.actc_lookback.update( - start=instant("2024-01-01"), - stop=instant("2024-12-31"), - value=True, - ) - parameters.gov.irs.credits.ctc.refundable.individual_max.update( - start=instant("2024-01-01"), - stop=instant("2024-12-31"), - value=1_900, - ) - parameters.gov.irs.credits.ctc.amount.base[0].amount.update( - start=instant("2024-01-01"), - stop=instant("2024-12-31"), - value=2_100, - ) - return parameters - - self.modify_parameters(modify_parameters) - - return wyden_smith_ctc_2024_parameter_reform - - -def _obbba_no_tax_on_tips_repeal_parameter_reform(): - from policyengine_core.periods import instant - from policyengine_us.model_api import Reform - - class obbba_no_tax_on_tips_repeal_parameter_reform(Reform): - def apply(self): - def modify_parameters(parameters): - parameters.gov.irs.deductions.tip_income.cap.update( - start=instant("2026-01-01"), - stop=instant("2026-12-31"), - value=0, - ) - return parameters - - self.modify_parameters(modify_parameters) - - return obbba_no_tax_on_tips_repeal_parameter_reform - - -def _obbba_no_tax_on_overtime_repeal_parameter_reform(): - from policyengine_core.periods import instant - from policyengine_us.model_api import Reform - - class obbba_no_tax_on_overtime_repeal_parameter_reform(Reform): - def apply(self): - def modify_parameters(parameters): - caps = parameters.gov.irs.deductions.overtime_income.cap - for filing_status in ( - "JOINT", - "HEAD_OF_HOUSEHOLD", - "SURVIVING_SPOUSE", - "SINGLE", - "SEPARATE", - ): - getattr(caps, filing_status).update( - start=instant("2026-01-01"), - stop=instant("2026-12-31"), - value=0, - ) - return parameters - - self.modify_parameters(modify_parameters) - - return obbba_no_tax_on_overtime_repeal_parameter_reform - - -def _obbba_senior_deduction_repeal_parameter_reform(): - from policyengine_core.periods import instant - from policyengine_us.model_api import Reform - - class obbba_senior_deduction_repeal_parameter_reform(Reform): - def apply(self): - def modify_parameters(parameters): - parameters.gov.irs.deductions.senior_deduction.amount.update( - start=instant("2026-01-01"), - stop=instant("2026-12-31"), - value=0, - ) - return parameters - - self.modify_parameters(modify_parameters) - - return obbba_senior_deduction_repeal_parameter_reform - - -def _kypa_childless_eitc_parameter_reform(): - from policyengine_core.periods import instant - from policyengine_us.model_api import Reform - - class kypa_childless_eitc_parameter_reform(Reform): - def apply(self): - def modify_parameters(parameters): - eitc = parameters.gov.irs.credits.eitc - start = instant("2026-01-01") - stop = instant("2035-12-31") - - eitc.max[0].amount.update(start=start, stop=stop, value=1_502) - eitc.phase_in_rate[0].amount.update( - start=start, - stop=stop, - value=0.153, - ) - eitc.phase_out.rate[0].amount.update( - start=start, - stop=stop, - value=0.153, - ) - eitc.phase_out.start[0].amount.update( - start=start, - stop=stop, - value=11_610, - ) - eitc.eligibility.age.min.update(start=start, stop=stop, value=19) - eitc.eligibility.age.min_student.update( - start=start, - stop=stop, - value=19, - ) - eitc.eligibility.age.max.update( - start=start, - stop=stop, - value=float("inf"), - ) - return parameters - - self.modify_parameters(modify_parameters) - - return kypa_childless_eitc_parameter_reform - - -def _reform_object(reform_id: str): - if reform_id == "halve_joint_eitc_phase_out_rate": - from policyengine_us.reforms.eitc.halve_joint_eitc_phase_out_rate import ( - halve_joint_eitc_phase_out_rate, - ) - - return halve_joint_eitc_phase_out_rate - if reform_id == "american_family_act_2025": - from policyengine_us.reforms.congress.afa.afa_other_dependent_credit import ( - afa_other_dependent_credit, - ) - - return afa_other_dependent_credit - if reform_id == "working_parents_tax_relief_act_2026": - from policyengine_us.reforms.congress.mcdonald_rivet.working_parents_tax_relief_act.working_parents_tax_relief_act import ( # noqa: E501 - working_parents_tax_relief_act, - ) - - return ( - working_parents_tax_relief_act, - _working_parents_tax_relief_parameter_reform(), - ) - if reform_id == "wyden_smith_ctc_2024": - from policyengine_us.reforms.congress.wyden_smith.ctc_expansion import ( - ctc_expansion, - ) - - return ( - ctc_expansion, - _wyden_smith_ctc_2024_parameter_reform(), - ) - if reform_id == "kypa_ctc_2026": - from policyengine_us.reforms.congress.afa.afa_other_dependent_credit import ( - afa_other_dependent_credit, - ) - - return afa_other_dependent_credit - if reform_id == "kypa_childless_eitc_2026": - return _kypa_childless_eitc_parameter_reform() - if reform_id == "obbba_no_tax_on_tips_repeal_2026": - return _obbba_no_tax_on_tips_repeal_parameter_reform() - if reform_id == "obbba_no_tax_on_overtime_repeal_2026": - return _obbba_no_tax_on_overtime_repeal_parameter_reform() - if reform_id == "obbba_senior_deduction_repeal_2026": - return _obbba_senior_deduction_repeal_parameter_reform() - raise ValueError(f"Unknown reform_id: {reform_id}") - - -@lru_cache(maxsize=64) -def _microsimulation(dataset: str, reform_id: str | None): - from policyengine_us import Microsimulation - - if reform_id is None: - return Microsimulation(dataset=dataset) - return Microsimulation(dataset=dataset, reform=_reform_object(reform_id)) - - -def _finite_float(value: Any) -> float | None: - try: - result = float(value) - except (TypeError, ValueError): - return None - return result if np.isfinite(result) else None - - -def _weighted_total( - sim, - *, - variable: str, - period: int, - entity: str, -) -> dict[str, Any]: - values = sim.calculate(variable, period=period, map_to=entity) - weights = sim.calculate("household_weight", period=period, map_to=entity) - value_array = np.asarray(values.values, dtype=float) - weight_array = np.asarray(weights.values, dtype=float) - return { - "total": _finite_float(np.sum(value_array * weight_array)), - "unweighted_mean": _finite_float(np.mean(value_array)), - "record_count": int(value_array.size), - "weight_sum": _finite_float(np.sum(weight_array)), - } - - -def _run_dataset_reform_comparison( - *, - dataset: str, - reform_id: str, - variable: str, - period: int, - entity: str, -) -> dict[str, Any]: - baseline = _microsimulation(dataset, None) - reformed = _microsimulation(dataset, reform_id) - baseline_total = _weighted_total( - baseline, - variable=variable, - period=period, - entity=entity, - ) - reformed_total = _weighted_total( - reformed, - variable=variable, - period=period, - entity=entity, - ) - baseline_value = baseline_total["total"] - reformed_value = reformed_total["total"] - delta = ( - reformed_value - baseline_value - if baseline_value is not None and reformed_value is not None - else None - ) - return { - "dataset": dataset, - "baseline": baseline_total, - "reform": reformed_total, - "delta": _finite_float(delta), - } - - -def _budget_effect_from_delta( - delta: float | None, - *, - rule: str, -) -> float | None: - if delta is None: - return None - # Positive budget effect means higher federal cost / lower federal revenue. - if rule == "credit_delta_is_cost": - return _finite_float(delta) - if rule == "tax_revenue_delta_is_negative_cost": - return _finite_float(-delta) - if rule == "repeal_tax_delta_is_provision_cost": - return _finite_float(delta) - return _finite_float(delta) - - -def _budget_gap( - model_effect: float | None, - external_estimate: Any, -) -> dict[str, float | None]: - estimate = _finite_float(external_estimate) - if model_effect is None or estimate is None: - return {"gap": None, "ratio": None} - return { - "gap": _finite_float(model_effect - estimate), - "ratio": _finite_float(model_effect / estimate) if estimate else None, - } - - -@router.get("/microplex/reform-comparison") -def microplex_reform_comparison( - reform_id: str = "american_family_act_2025", - variable: str | None = None, - period: int | None = None, -): - """Run the same PolicyEngine reform over us-data and Microplex H5s.""" - preset = _REFORM_PRESETS.get(reform_id) - if preset is None: - raise HTTPException( - status_code=400, - detail=f"Unknown reform_id. Available: {sorted(_REFORM_PRESETS)}", - ) - selected_variable = variable or str(preset["variable"]) - selected_period = int(period or preset["period"]) - entity = str(preset["entity"]) - - latest = _latest_microplex_policyengine_dataset() - if latest is None: - return { - "available": False, - "reason": ( - "No configured Microplex run bundle with policyengine_us.h5 was " - "found. Set MICROPLEX_ARTIFACT_ROOTS or MICROPLEX_ARTIFACT_ROOT." - ), - "reform": None, - "period": selected_period, - "available_reforms": list(_REFORM_PRESETS.values()), - "outcomes": [], - } - - microplex_dataset = str(latest["policyengine_dataset_path"]) - us_data_dataset = os.environ.get( - "MICROSIM_US_DATASET", - "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5", - ) - cache_key = ( - microplex_dataset, - us_data_dataset, - reform_id, - selected_variable, - selected_period, - ) - cached = _REFORM_COMPARISON_CACHE.get(cache_key) - if cached and time.time() - cached[0] < _REFORM_COMPARISON_TTL_SECONDS: - return cached[1] - - started = time.time() - try: - us_data = _run_dataset_reform_comparison( - dataset=us_data_dataset, - reform_id=reform_id, - variable=selected_variable, - period=selected_period, - entity=entity, - ) - microplex_result = _run_dataset_reform_comparison( - dataset=microplex_dataset, - reform_id=reform_id, - variable=selected_variable, - period=selected_period, - entity=entity, - ) - except Exception as exc: - logger.exception("Failed to run Microplex reform comparison") - raise HTTPException( - status_code=500, - detail=f"Failed to run reform comparison: {exc}", - ) from exc - - us_delta = us_data["delta"] - microplex_delta = microplex_result["delta"] - delta_gap = ( - microplex_delta - us_delta - if microplex_delta is not None and us_delta is not None - else None - ) - delta_ratio = ( - microplex_delta / us_delta - if microplex_delta is not None and us_delta not in (None, 0) - else None - ) - payload = _scrub( - { - "available": True, - "runtime_seconds": time.time() - started, - "period": selected_period, - "available_reforms": list(_REFORM_PRESETS.values()), - "reform": { - "id": reform_id, - "label": preset["label"], - "description": preset["description"], - "source_url": preset["source_url"], - }, - "microplex_bundle": { - "artifact_id": latest.get("artifact_id"), - "artifact_dir": latest.get("artifact_dir"), - "policyengine_dataset_path": microplex_dataset, - }, - "us_data_dataset": us_data_dataset, - "outcomes": [ - { - "variable": selected_variable, - "entity": entity, - "unit": preset["unit"], - "us_data": us_data, - "microplex": microplex_result, - "delta_gap": _finite_float(delta_gap), - "microplex_delta_as_share_of_us_data": _finite_float( - delta_ratio - ), - } - ], - } - ) - _REFORM_COMPARISON_CACHE[cache_key] = (time.time(), payload) - return payload - - -@router.get("/microplex/budget-benchmarks") -def microplex_budget_benchmarks(compute_live: bool = False) -> dict[str, Any]: - """Return budget-score benchmark rows for us-data and Microplex. - - This endpoint intentionally distinguishes exact live comparisons from - external-only benchmark rows. The dashboard should only interpret a row as - a CBO/JCT validation when a matching live reform preset and comparable - external estimate are both present. - """ - latest = _latest_microplex_policyengine_dataset() - microplex_dataset = ( - str(latest["policyengine_dataset_path"]) - if isinstance(latest, dict) and latest.get("policyengine_dataset_path") - else "" - ) - us_data_dataset = os.environ.get( - "MICROSIM_US_DATASET", - "hf://policyengine/policyengine-us-data/enhanced_cps_2024.h5", - ) - cache_key = ( - microplex_dataset or "no-microplex-h5", - us_data_dataset, - len(_BUDGET_BENCHMARKS), - int(compute_live), - ) - cached = _BUDGET_BENCHMARK_CACHE.get(cache_key) - if cached and time.time() - cached[0] < _BUDGET_BENCHMARK_TTL_SECONDS: - return cached[1] - - started = time.time() - rows: list[dict[str, Any]] = [] - errors: list[dict[str, str]] = [] - - for benchmark in _BUDGET_BENCHMARKS: - row = { - "id": benchmark["id"], - "title": benchmark["title"], - "policy_area": benchmark["policy_area"], - "benchmark_period": benchmark["benchmark_period"], - "comparison_status": benchmark["comparison_status"], - "budget_effect_rule": benchmark["budget_effect_rule"], - "notes": benchmark["notes"], - "external_estimates": benchmark["external_estimates"], - "live": { - "available": False, - "reason": None, - "reform": None, - "period": None, - "outcome_variable": None, - "outcome_entity": None, - "unit": None, - "us_data": None, - "microplex": None, - "microplex_budget_effect_as_share_of_us_data": None, - "budget_effect_gap": None, - }, - } - live_reform_id = benchmark.get("live_reform_id") - preset = ( - _REFORM_PRESETS.get(str(live_reform_id)) - if live_reform_id is not None - else None - ) - if preset is None: - row["live"]["reason"] = "No matching live reform preset is wired yet." - rows.append(row) - continue - if not compute_live: - row["live"]["reason"] = ( - "Live microsim compute is deferred. Set compute_live=true to " - "run this benchmark over us-data and Microplex." - ) - rows.append(row) - continue - if not microplex_dataset: - row["live"]["reason"] = ( - "No configured Microplex run bundle with policyengine_us.h5 was found." - ) - rows.append(row) - continue - - variable = str(preset["variable"]) - period = int(preset["period"]) - entity = str(preset["entity"]) - try: - us_data = _run_dataset_reform_comparison( - dataset=us_data_dataset, - reform_id=str(live_reform_id), - variable=variable, - period=period, - entity=entity, - ) - microplex_result = _run_dataset_reform_comparison( - dataset=microplex_dataset, - reform_id=str(live_reform_id), - variable=variable, - period=period, - entity=entity, - ) - except Exception as exc: - logger.exception("Failed to run budget benchmark %s", live_reform_id) - message = f"Failed to run live microsim: {exc}" - row["live"]["reason"] = message - errors.append({"benchmark_id": str(benchmark["id"]), "error": message}) - rows.append(row) - continue - - us_budget_effect = _budget_effect_from_delta( - us_data["delta"], - rule=str(benchmark["budget_effect_rule"]), - ) - microplex_budget_effect = _budget_effect_from_delta( - microplex_result["delta"], - rule=str(benchmark["budget_effect_rule"]), - ) - budget_effect_gap = ( - microplex_budget_effect - us_budget_effect - if microplex_budget_effect is not None and us_budget_effect is not None - else None - ) - budget_effect_ratio = ( - microplex_budget_effect / us_budget_effect - if microplex_budget_effect is not None - and us_budget_effect not in (None, 0) - else None - ) - external_estimates = [] - for estimate in benchmark["external_estimates"]: - us_gap = _budget_gap(us_budget_effect, estimate.get("estimate")) - microplex_gap = _budget_gap( - microplex_budget_effect, - estimate.get("estimate"), - ) - external_estimates.append( - { - **estimate, - "comparable_to_live_annual_result": bool( - estimate.get("comparable_to_live_annual_result") - ), - "us_data_gap": us_gap["gap"], - "us_data_ratio": us_gap["ratio"], - "microplex_gap": microplex_gap["gap"], - "microplex_ratio": microplex_gap["ratio"], - } - ) - - row["external_estimates"] = external_estimates - row["live"] = { - "available": True, - "reason": None, - "reform": { - "id": preset["id"], - "label": preset["label"], - "description": preset["description"], - "source_url": preset["source_url"], - }, - "period": period, - "outcome_variable": variable, - "outcome_entity": entity, - "unit": preset["unit"], - "us_data": { - **us_data, - "budget_effect": us_budget_effect, - }, - "microplex": { - **microplex_result, - "budget_effect": microplex_budget_effect, - }, - "microplex_budget_effect_as_share_of_us_data": _finite_float( - budget_effect_ratio - ), - "budget_effect_gap": _finite_float(budget_effect_gap), - } - rows.append(row) - - payload = _scrub( - { - "available": True, - "runtime_seconds": time.time() - started, - "generated_at_unix": time.time(), - "sign_convention": ( - "Positive budget effect means higher federal cost or lower " - "federal revenue. For counterfactual repeal rows, the model " - "uses the tax increase from removing one current-law provision " - "as the cost of keeping that provision." - ), - "comparison_caveat": ( - "Clean scored rows compare one modeled provision at a time. " - "External decade scores are context unless the row explicitly " - "marks the annual estimate as comparable to the live result. " - "OBBBA JCT line items are shown as context because they are " - "baseline- and package-order-sensitive. " - "Live income-tax microsims are expensive on the full dataset, " - "so this endpoint only computes them when compute_live=true." - ), - "compute_live": compute_live, - "us_data_dataset": us_data_dataset, - "microplex_bundle": { - "available": bool(microplex_dataset), - "artifact_id": latest.get("artifact_id") if isinstance(latest, dict) else None, - "artifact_dir": latest.get("artifact_dir") if isinstance(latest, dict) else None, - "policyengine_dataset_path": microplex_dataset or None, - }, - "rows": rows, - "errors": errors, - } - ) - _BUDGET_BENCHMARK_CACHE[cache_key] = (time.time(), payload) - return payload - - -def _load_target_diagnostics(latest_bundle: dict[str, Any]) -> dict[str, Any]: - path_text = latest_bundle.get("target_diagnostics_path") - if not path_text: - return { - "available": False, - "path": None, - "summary": {}, - "total_targets": 0, - "display_limit": 100, - "targets": [], - } - path = Path(str(path_text)) - payload = _read_json_file(path) - if not isinstance(payload, dict): - return { - "available": False, - "path": str(path), - "summary": {}, - "total_targets": 0, - "display_limit": 100, - "targets": [], - } - targets = payload.get("targets") - rows = targets if isinstance(targets, list) else [] - display_limit = 100 - summary = payload.get("summary") - return { - "available": True, - "path": str(path), - "diagnostic_schema_version": payload.get("diagnostic_schema_version"), - "metric": payload.get("metric"), - "period": payload.get("period"), - "baseline_dataset": payload.get("baseline_dataset"), - "candidate_dataset": payload.get("candidate_dataset"), - "dataset_labels": payload.get("dataset_labels", {}), - "summary": summary if isinstance(summary, dict) else {}, - "total_targets": len(rows), - "display_limit": display_limit, - "targets": rows[:display_limit], - } - - -def _load_target_diagnostics_payload( - latest_bundle: dict[str, Any], -) -> tuple[Path | None, dict[str, Any] | None]: - path_text = latest_bundle.get("target_diagnostics_path") - if not path_text: - return None, None - path = Path(str(path_text)) - payload = _read_json_file(path) - return path, payload if isinstance(payload, dict) else None - - -def _row_matches_text(row: dict[str, Any], search: str) -> bool: - needle = search.lower() - fields = [ - "target_id", - "target_name", - "family", - "target_family", - "geography", - "state", - "variable", - "entity", - ] - return any(needle in str(row.get(field, "")).lower() for field in fields) - - -def _estimate(value: Any) -> float | None: - if isinstance(value, bool): - return None - if isinstance(value, int | float) and np.isfinite(value): - return float(value) - return None - - -def _relative_difference( - numerator: float | None, - denominator: float | None, -) -> float | None: - if numerator is None or denominator in (None, 0): - return None - return numerator / abs(denominator) - - -def _enrich_target_diagnostic_row(row: dict[str, Any]) -> dict[str, Any]: - target = _estimate(row.get("target_value")) - us_data = _estimate(row.get("us_data_aggregate", row.get("from_estimate"))) - microplex = _estimate(row.get("microplex_aggregate", row.get("to_estimate"))) - delta_abs_error = _estimate(row.get("delta_absolute_error")) - microplex_vs_target = ( - microplex - target if microplex is not None and target is not None else None - ) - us_data_vs_target = ( - us_data - target if us_data is not None and target is not None else None - ) - microplex_vs_us_data = ( - microplex - us_data - if microplex is not None and us_data is not None - else None - ) - return { - **row, - "microplex_vs_target": microplex_vs_target, - "us_data_vs_target": us_data_vs_target, - "microplex_vs_us_data": microplex_vs_us_data, - "microplex_vs_target_relative": _relative_difference( - microplex_vs_target, - target, - ), - "us_data_vs_target_relative": _relative_difference( - us_data_vs_target, - target, - ), - "microplex_vs_us_data_relative": _relative_difference( - microplex_vs_us_data, - us_data, - ), - "closer_dataset": ( - None - if delta_abs_error is None - else "microplex" - if delta_abs_error < 0 - else "us-data" - if delta_abs_error > 0 - else "tie" - ), - } - - -def _sort_target_diagnostic_rows( - rows: list[dict[str, Any]], - *, - sort_by: str | None, - sort_dir: str | None, -) -> list[dict[str, Any]]: - if not sort_by: - return rows - descending = sort_dir == "desc" - - def compare(left: dict[str, Any], right: dict[str, Any]) -> int: - left_value = _row_sort_value(left.get(sort_by)) - right_value = _row_sort_value(right.get(sort_by)) - if left_value is None and right_value is None: - return 0 - if left_value is None: - return 1 - if right_value is None: - return -1 - if isinstance(left_value, float) and isinstance(right_value, float): - result = (left_value > right_value) - (left_value < right_value) - else: - result = (str(left_value) > str(right_value)) - ( - str(left_value) < str(right_value) - ) - return -result if descending else result - - return sorted(rows, key=cmp_to_key(compare)) - - -def _row_sort_value(value: Any) -> float | str | None: - if value is None: - return None - numeric_value = _estimate(value) - if numeric_value is not None: - return numeric_value - if isinstance(value, int | float): - return None - return str(value).lower() - - -@router.get("/microplex/target-diagnostics") -def microplex_target_diagnostics( - limit: int = 100, - offset: int = 0, - family: str | None = None, - state: str | None = None, - geo_level: str | None = None, - microplex_target_direction: str | None = None, - supported: bool | None = None, - in_loss: bool | None = None, - search: str | None = None, - sort_by: str | None = None, - sort_dir: str | None = None, -): - """Return paginated full Microplex target diagnostics rows.""" - limit = max(1, min(int(limit), 500)) - offset = max(0, int(offset)) - latest = _discover_configured_run_bundles().get("latest_run_bundle") - if not isinstance(latest, dict): - return { - "available": False, - "reason": ( - "No configured Microplex run bundle was found. Set " - "MICROPLEX_ARTIFACT_ROOTS or MICROPLEX_ARTIFACT_ROOT." - ), - "targets": [], - "total_targets": 0, - "filtered_total": 0, - "limit": limit, - "offset": offset, - } - - path, payload = _load_target_diagnostics_payload(latest) - if payload is None: - return { - "available": False, - "reason": "No readable pe_native_target_diagnostics.json was found.", - "path": str(path) if path is not None else None, - "microplex_bundle": { - "artifact_id": latest.get("artifact_id"), - "artifact_dir": latest.get("artifact_dir"), - }, - "targets": [], - "total_targets": 0, - "filtered_total": 0, - "limit": limit, - "offset": offset, - } - - rows = payload.get("targets") - all_rows = ( - [_enrich_target_diagnostic_row(row) for row in rows if isinstance(row, dict)] - if isinstance(rows, list) - else [] - ) - filtered = all_rows - if family: - filtered = [ - row - for row in filtered - if str(row.get("family") or row.get("target_family") or "") == family - ] - if state: - state_upper = state.upper() - filtered = [ - row - for row in filtered - if str(row.get("state") or "").upper() == state_upper - ] - if geo_level: - filtered = [ - row - for row in filtered - if str(row.get("geo_level") or "") == geo_level - ] - if microplex_target_direction: - filtered = [ - row - for row in filtered - if ( - (relative := _estimate(row.get("microplex_vs_target_relative"))) - is not None - and ( - ( - microplex_target_direction == "above" - and relative > 0 - ) - or ( - microplex_target_direction == "below" - and relative < 0 - ) - or ( - microplex_target_direction == "near" - and abs(relative) <= 0.05 - ) - or microplex_target_direction - not in {"above", "below", "near"} - ) - ) - ] - if supported is not None: - filtered = [ - row for row in filtered if row.get("supported_by_microplex") is supported - ] - if in_loss is not None: - filtered = [row for row in filtered if row.get("in_loss") is in_loss] - if search: - filtered = [row for row in filtered if _row_matches_text(row, search)] - filtered = _sort_target_diagnostic_rows( - filtered, - sort_by=sort_by, - sort_dir=sort_dir, - ) - - page = filtered[offset : offset + limit] - summary = payload.get("summary") - return _scrub( - { - "available": True, - "path": str(path) if path is not None else None, - "microplex_bundle": { - "artifact_id": latest.get("artifact_id"), - "artifact_dir": latest.get("artifact_dir"), - }, - "diagnostic_schema_version": payload.get("diagnostic_schema_version"), - "metric": payload.get("metric"), - "period": payload.get("period"), - "baseline_dataset": payload.get("baseline_dataset"), - "candidate_dataset": payload.get("candidate_dataset"), - "dataset_labels": payload.get("dataset_labels", {}), - "summary": summary if isinstance(summary, dict) else {}, - "total_targets": len(all_rows), - "filtered_total": len(filtered), - "unfiltered_total_targets": len(all_rows), - "returned": len(page), - "limit": limit, - "offset": offset, - "has_next": offset + limit < len(filtered), - "filters": { - "family": family, - "state": state, - "geo_level": geo_level, - "microplex_target_direction": microplex_target_direction, - "supported": supported, - "in_loss": in_loss, - "search": search, - "sort_by": sort_by, - "sort_dir": sort_dir, - }, - "targets": page, - } - ) - - -def _load_bundle_native_scores(latest_bundle: dict[str, Any]) -> dict[str, Any] | None: - path_text = latest_bundle.get("native_scores_path") - if not path_text: - return None - path = Path(str(path_text)) - payload = _read_json_file(path) - if not isinstance(payload, dict): - return None - - summary = payload.get("summary") - if not isinstance(summary, dict): - summary = payload.get("broad_loss") - if not isinstance(summary, dict): - summary = payload - - scores = { - "available": True, - "metric": payload.get("metric") or summary.get("metric"), - "period": payload.get("period") or summary.get("period"), - "baseline_enhanced_cps_native_loss": summary.get( - "baseline_enhanced_cps_native_loss" - ), - "candidate_enhanced_cps_native_loss": summary.get( - "candidate_enhanced_cps_native_loss" - ), - "enhanced_cps_native_loss_delta": summary.get( - "enhanced_cps_native_loss_delta" - ), - "baseline_unweighted_msre": summary.get("baseline_unweighted_msre"), - "candidate_unweighted_msre": summary.get("candidate_unweighted_msre"), - "unweighted_msre_delta": summary.get("unweighted_msre_delta"), - "candidate_beats_baseline": summary.get("candidate_beats_baseline"), - "n_targets_total": summary.get("n_targets_total"), - "n_targets_kept": summary.get("n_targets_kept"), - "n_national_targets": summary.get("n_national_targets"), - "n_state_targets": summary.get("n_state_targets"), - "n_targets_bad_dropped": summary.get("n_targets_bad_dropped"), - "n_targets_zero_dropped": summary.get("n_targets_zero_dropped"), - "source": "configured_run_bundle", - "source_path": str(path), - "artifact_id": latest_bundle.get("artifact_id"), - } - if scores["candidate_beats_baseline"] is None: - delta = scores["enhanced_cps_native_loss_delta"] - if isinstance(delta, int | float): - scores["candidate_beats_baseline"] = float(delta) < 0.0 - return scores - - -@router.get("/microplex") -def microplex_overview() -> dict: - """Return a consolidated Microplex target-performance payload. - - Pulls three committed JSONs from PolicyEngine/microplex-us via raw - GitHub (no auth required). The response is structured for a single - dashboard page; consumers should pluck what they need. - """ - parity = _fetch_json(_PARITY_PATH) - regression = _fetch_json(_REGRESSION_SUMMARY_PATH) - drilldown = _fetch_json(_IRS_DRILLDOWN_PATH) - configured_runs = _discover_configured_run_bundles() - latest_bundle = configured_runs.get("latest_run_bundle") or {} - target_rows_available = bool(latest_bundle.get("target_diagnostics_exists")) - target_diagnostics = _load_target_diagnostics(latest_bundle) - - # Pull the target-oracle headline numbers up to a flat shape the frontend - # can render without spelunking. Leave the raw payload available too. - headline = {} - ph = parity.get("comparison", {}).get("policyengineHarness") or {} - if ph.get("isPolicyEngineComparison"): - headline = { - "baseline_label": parity.get("baselineSlice", {}).get("baselineLabel"), - "candidate_label": parity.get("baselineSlice", {}).get("candidateLabel"), - "calibration_target_profile": parity.get("baselineSlice", {}).get( - "calibrationTargetProfile" - ), - "n_synthetic": parity.get("baselineSlice", {}) - .get("comparisonMetadata", {}) - .get("n_synthetic"), - "target_period": parity.get("baselineSlice", {}).get("targetPeriod"), - "baseline_composite_parity_loss": ph.get("baseline_composite_parity_loss"), - "candidate_composite_parity_loss": ph.get("candidate_composite_parity_loss"), - "composite_parity_loss_delta": ph.get("composite_parity_loss_delta"), - "baseline_mean_abs_relative_error": ph.get("baseline_mean_abs_relative_error"), - "candidate_mean_abs_relative_error": ph.get("candidate_mean_abs_relative_error"), - "mean_abs_relative_error_delta": ph.get("mean_abs_relative_error_delta"), - "slice_win_rate": ph.get("slice_win_rate"), - "supported_target_rate": ph.get("supported_target_rate"), - "target_win_rate": ( - ph.get("tag_summaries", {}) - .get("all_targets", {}) - .get("target_win_rate") - ), - "tag_summaries": ph.get("tag_summaries", {}), - } - parity_native_scores = ( - parity.get("comparison", {}).get("policyengineNativeScores") or {} - ) - bundle_native_scores = _load_bundle_native_scores(latest_bundle) - native_scores = bundle_native_scores or parity_native_scores - - return _scrub({ - "source_repo": "PolicyEngine/microplex-us", - "source_artifacts": [ - { - "name": name, - "path": path, - "url": f"{_GITHUB_RAW}/{path}", - } - for name, path in _ARTIFACTS.items() - ], - "limitations": [ - "Only committed microplex-us summary JSON artifacts are public.", - "Newer Microplex run bundles write pe_native_target_diagnostics.json, but those bundles are generated artifacts, not committed public JSONs.", - "This is aggregate Microplex target-oracle reporting, not the full row-level target performance table.", - ], - "newer_runs": { - "current_reader": "public_github_committed_summary_jsons", - "public_branch": "PolicyEngine/microplex-us main", - "run_bundle_manifest_key": _RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - "run_bundle_path_hint": _RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - "legacy_static_dashboard_path": _LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH, - "required_to_load_newer_runs": ( - "Point the dashboard at a generated Microplex artifact root, " - "publish the run-bundle JSONs, or expose the run index/artifacts " - "through an authenticated artifact service." - ), - "not_loaded_reason": ( - "The committed public repo only contains summary JSONs; this " - "process can only see newer runs when MICROPLEX_ARTIFACT_ROOTS " - "or MICROPLEX_ARTIFACT_ROOT points at generated run bundles, " - "or when an artifact store is wired in." - ), - "configured_run_discovery": configured_runs, - }, - "repo_structure": { - "canonical_stage_count": 9, - "current_commit_public_artifact_count": len(_ARTIFACTS), - "analysis_modes": [ - "microplex_vs_target_oracle", - "microplex_vs_us_data_comparator", - "run_to_run_microplex_comparison", - ], - "generated_artifacts": _GENERATED_ARTIFACT_CONTRACT, - "full_target_diagnostics": { - "available_in_committed_repo": False, - "expected_path": _RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - "run_level_path": _RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - "manifest_key": _RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - "legacy_static_dashboard_path": _LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH, - "static_dashboard_default_url": ( - f"../{_LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH}" - ), - "producer_command": ( - "Run the Microplex PE-US-data rebuild/native audit pipeline; " - "newer runs record manifest.artifacts." - f"{_RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY} = " - f"{_RUN_LEVEL_TARGET_DIAGNOSTICS_PATH}." - ), - "row_fields": _TARGET_DIAGNOSTIC_ROW_FIELDS, - "primary_use": ( - "Standalone Microplex aggregate-vs-target diagnostics; " - "us-data baseline fields are optional comparator context." - ), - }, - "run_index": { - "path_hint": "run_index.duckdb", - "query_helpers": [ - "list_us_microplex_target_delta_rows", - "compare_us_microplex_target_delta_rows", - "select_us_microplex_frontier_index_row", - ], - }, - }, - "artifact_id": latest_bundle.get("artifact_id") or parity.get("artifactId"), - "verdict": parity.get("verdict"), - "headline": headline, - "native_scores": { - "available": native_scores.get("available"), - "source": native_scores.get("source") or "public_committed_parity", - "source_path": native_scores.get("source_path"), - "artifact_id": native_scores.get("artifact_id"), - "metric": native_scores.get("metric"), - "period": native_scores.get("period"), - "baseline_enhanced_cps_native_loss": native_scores.get( - "baseline_enhanced_cps_native_loss" - ), - "candidate_enhanced_cps_native_loss": native_scores.get( - "candidate_enhanced_cps_native_loss" - ), - "enhanced_cps_native_loss_delta": native_scores.get( - "enhanced_cps_native_loss_delta" - ), - "baseline_unweighted_msre": native_scores.get("baseline_unweighted_msre"), - "candidate_unweighted_msre": native_scores.get("candidate_unweighted_msre"), - "unweighted_msre_delta": native_scores.get("unweighted_msre_delta"), - "candidate_beats_baseline": native_scores.get("candidate_beats_baseline"), - "n_targets_total": native_scores.get("n_targets_total"), - "n_targets_kept": native_scores.get("n_targets_kept"), - "n_national_targets": native_scores.get("n_national_targets"), - "n_state_targets": native_scores.get("n_state_targets"), - "n_targets_bad_dropped": native_scores.get("n_targets_bad_dropped"), - "n_targets_zero_dropped": native_scores.get("n_targets_zero_dropped"), - "target_rows_available": target_rows_available, - "full_target_diagnostics_path": ( - latest_bundle.get("target_diagnostics_path") - or _RUN_LEVEL_TARGET_DIAGNOSTICS_PATH - ), - "full_target_diagnostics_manifest_key": ( - _RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY - ), - }, - "target_diagnostics": target_diagnostics, - "regression_summary": { - "total_scored_runs": regression.get("totalScoredRuns"), - "total_audited_runs": regression.get("totalAuditedRuns"), - "best_runs": regression.get("bestRuns", [])[:10], - "worst_runs": regression.get("worstRuns", [])[:10], - "largest_family_counts": regression.get("largestFamilyCounts", {}), - "top3_family_counts": regression.get("top3FamilyCounts", {}), - "target_counts_from_audits": regression.get("targetCountsFromAudits", {}), - }, - "irs_drilldown": { - "family": drilldown.get("family"), - "audits_where_family_leads": drilldown.get("auditsWhereFamilyLeads"), - "audits_with_matching_targets": drilldown.get( - "auditsWithMatchingTargets" - ), - "lead_audits": drilldown.get("leadAudits", [])[:10], - "lead_target_counts": drilldown.get("leadTargetCounts", {}), - "lead_filing_status_gap_summary": drilldown.get( - "leadFilingStatusGapSummary" - ), - "lead_mfs_agi_gap_summary": drilldown.get("leadMFSAgiGapSummary"), - }, - }) diff --git a/backend/routes/nodes.py b/backend/routes/nodes.py deleted file mode 100644 index 94fc7a2..0000000 --- a/backend/routes/nodes.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Node-variable inventory. - -A "node variable" is a leaf in the policyengine_us variable tree: it has no -formula and is not built up from other variables via `adds`/`subtracts`. -Uprating is allowed — that's just CPI/wage projection of a data value, not -derivation. These are the variables the microsim cannot compute on its own; -their values have to come from the underlying dataset or a calibration target. -This view lets users see, for the loaded run, which leaves actually carry a -calibration target and which are left to whatever upstream data provides. -""" - -from __future__ import annotations - -from fastapi import APIRouter, Depends - -from backend.state import AppState, get_state - -router = APIRouter() - - -def _tax_benefit_system(state: AppState): - """The state's sim_service is either a SimService wrapper (pkl mode) or - a raw Microsimulation (dataset mode). Both expose tax_benefit_system, - one via ._sim and the other directly.""" - sim = state.sim_service - if sim is None: - return None - return getattr(sim, "_sim", sim).tax_benefit_system - - -def _is_leaf(var) -> bool: - """No way to compute this variable from others — it must come from data. - - Uprating is permitted: a parameter-driven yearly projection of a stored - value isn't derivation, just inflation/wage indexing. Many calibration - targets (e.g. unemployment_compensation, dividend_income, - social_security_*) have no formula and no adds/subtracts — only uprating. - Treating them as non-leaves would hide the bulk of what the loss actually - pins down at the input layer. - """ - if getattr(var, "formulas", None): - return False - if getattr(var, "adds", None): - return False - if getattr(var, "subtracts", None): - return False - return True - - -@router.get("/nodes") -def list_nodes(state: AppState = Depends(get_state)) -> dict: - """Return every leaf input variable and whether it has a calibration target. - - is_calibrated is true iff the variable name appears in the loaded run's - targets table at any geo level / constraint set. - """ - tbs = _tax_benefit_system(state) - if tbs is None: - return {"items": [], "total": 0, "n_calibrated": 0} - - calibrated_vars: set[str] = set() - df = state.targets_enriched - if not df.empty and "variable" in df.columns: - calibrated_vars = set(df["variable"].dropna().astype(str).unique()) - - items = [] - for name, var in tbs.variables.items(): - if not _is_leaf(var): - continue - items.append({ - "name": name, - "label": getattr(var, "label", None) or name, - "entity": var.entity.key, - "value_type": var.value_type.__name__, - "definition_period": getattr(var, "definition_period", None), - "documentation": getattr(var, "documentation", None) or None, - "is_calibrated": name in calibrated_vars, - }) - items.sort(key=lambda r: r["name"]) - - n_cal = sum(1 for r in items if r["is_calibrated"]) - return { - "items": items, - "total": len(items), - "n_calibrated": n_cal, - } diff --git a/backend/routes/pipeline.py b/backend/routes/pipeline.py deleted file mode 100644 index fc1052e..0000000 --- a/backend/routes/pipeline.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Endpoints serving the extracted pipeline DAG + per-stage deep-dives. - -The data is committed under backend/data/pipeline/. Run the AST extractor -(`backend/scripts/extract_pipeline_dag.py`) to refresh nodes.json when -policyengine_us_data is upgraded; per-stage markdown is hand- or -agent-authored. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -from fastapi import APIRouter, HTTPException - -router = APIRouter() - -DATA_ROOT = Path(__file__).resolve().parents[1] / "data" / "pipeline" -DEFAULT_PIPELINE_ID = "us-data" - -PIPELINES = { - "us-data": { - "label": "policyengine-us-data", - "description": "Extracted @pipeline_node DAG from policyengine_us_data.", - "nodes_path": DATA_ROOT / "nodes.json", - "docs_dir": DATA_ROOT / "stages", - "stage_labels": { - "1_build_datasets": "1. Build datasets", - "2_build_calibration_package": "2. Build calibration package", - "3_fit_weights": "3. Fit weights", - "4_build_outputs": "4. Build outputs", - "5_validate_and_promote_release": "5. Validate & promote release", - }, - }, - "microplex-us": { - "label": "Microplex-US", - "description": ( - "Curated US Microplex flow from source fusion through " - "PolicyEngine oracle evaluation and published diagnostics." - ), - "nodes_path": DATA_ROOT / "microplex.json", - "docs_dir": DATA_ROOT / "microplex" / "stages", - "stage_labels": { - "01_run_profile": "1. Run profile", - "02_source_loading": "2. Source loading", - "03_source_planning": "3. Source planning", - "04_seed_scaffold": "4. Seed scaffold", - "05_donor_integration_synthesis": "5. Donor integration", - "06_policyengine_entities": "6. PolicyEngine entities", - "07_calibration": "7. Calibration", - "08_dataset_assembly": "8. Dataset assembly", - "09_validation_benchmarking": "9. Validation & benchmarking", - }, - }, -} - - -def _pipeline_config(pipeline_id: str | None) -> dict: - selected = pipeline_id or DEFAULT_PIPELINE_ID - config = PIPELINES.get(selected) - if config is None: - raise HTTPException( - status_code=404, - detail=f"Unknown pipeline '{selected}'", - ) - return {"id": selected, **config} - - -def _load_nodes(pipeline_id: str | None = None) -> dict: - config = _pipeline_config(pipeline_id) - path = config["nodes_path"] - if not path.exists(): - raise HTTPException( - status_code=503, - detail=( - "Pipeline DAG hasn't been extracted yet. Run " - "`python backend/scripts/extract_pipeline_dag.py`." - ), - ) - return json.loads(path.read_text()) - - -@router.get("/pipeline") -def get_pipeline( - pipeline_id: str | None = None, - pipeline: str | None = None, -): - """Full DAG: nodes, edges, stats. Grouped by both stage (canonical) and - pathway (legacy) for the frontend.""" - selected_pipeline = pipeline_id or pipeline or DEFAULT_PIPELINE_ID - config = _pipeline_config(selected_pipeline) - payload = _load_nodes(selected_pipeline) - nodes = payload["nodes"] - - # Group by stage (canonical 5-stage taxonomy) - stages: dict[str, list[str]] = {} - for n in nodes: - sid = n.get("stage_id") or "(unknown)" - stages.setdefault(sid, []).append(n["id"]) - - # Group by pathway (legacy; kept so existing pathway docs still surface) - pathways: dict[str, list[str]] = {} - for n in nodes: - for p in (n.get("pathways") or ["(none)"]): - pathways.setdefault(p, []).append(n["id"]) - - stages_dir = config["docs_dir"] - has_doc: dict[str, bool] = {} - if stages_dir.exists(): - for path in stages_dir.glob("*.md"): - has_doc[path.stem] = True - - stage_summary = [ - { - "id": sid, - "label": config["stage_labels"].get(sid, sid), - "node_count": len(ids), - "has_doc": has_doc.get(sid, False), - } - for sid, ids in sorted(stages.items()) - ] - pathway_summary = [ - { - "id": p, - "label": p.replace("_", " ").title(), - "node_count": len(ids), - "has_doc": has_doc.get(p, False), - } - for p, ids in sorted(pathways.items()) - ] - return { - "pipeline_id": selected_pipeline, - "pipeline_label": payload.get("pipeline_label", config["label"]), - "description": payload.get("description", config["description"]), - "source_repo": payload.get("source_repo"), - "source_urls": payload.get("source_urls", []), - "nodes": nodes, - "edges": payload.get("edges", []), - "unproduced_artifacts": payload.get("unproduced_artifacts", []), - "stats": payload.get("stats", {}), - "stages": stage_summary, - "pathways": pathway_summary, - } - - -@router.get("/pipeline/stages/{stage_id}") -def get_stage( - stage_id: str, - pipeline_id: str | None = None, - pipeline: str | None = None, -): - """Markdown deep-dive for a single pathway/stage.""" - selected_pipeline = pipeline_id or pipeline or DEFAULT_PIPELINE_ID - config = _pipeline_config(selected_pipeline) - safe = stage_id.replace("/", "").replace("..", "") - md_path = config["docs_dir"] / f"{safe}.md" - if not md_path.exists(): - raise HTTPException( - status_code=404, - detail=f"No deep-dive doc for stage '{stage_id}'", - ) - return { - "pipeline_id": selected_pipeline, - "stage_id": stage_id, - "markdown": md_path.read_text(encoding="utf-8"), - } diff --git a/backend/routes/runs.py b/backend/routes/runs.py deleted file mode 100644 index 67ececa..0000000 --- a/backend/routes/runs.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Endpoints for run discovery and dataset listing.""" - -from fastapi import APIRouter, HTTPException - -from backend.services import runs as runs_service - -router = APIRouter() - - -@router.get("/datasets") -def list_datasets(): - return [ - { - "id": d.id, - "label": d.label, - "repo_id": d.repo_id, - "primary_h5": d.primary_h5, - } - for d in runs_service.list_datasets() - ] - - -@router.get("/runs") -def list_runs(dataset: str): - try: - runs = runs_service.list_runs(dataset) - except KeyError as exc: - raise HTTPException(status_code=400, detail=str(exc)) - return [ - { - "dataset_id": r.dataset_id, - "run_id": r.run_id, - "label": r.label, - "last_modified": r.last_modified, - } - for r in runs - ] diff --git a/backend/routes/strata.py b/backend/routes/strata.py deleted file mode 100644 index bd2bea9..0000000 --- a/backend/routes/strata.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Strata browser endpoint.""" - -from fastapi import APIRouter, Depends, HTTPException -from sqlmodel import Session - -from backend.state import get_state -from backend.models import StratumDetail -from backend.services import db_service -from backend.state import AppState - -router = APIRouter() - - -@router.get("/{stratum_id}") -def get_stratum( - stratum_id: int, - state: AppState = Depends(get_state), -) -> StratumDetail: - if state.db_engine is None: - raise HTTPException(status_code=503, detail="No database connected") - - with Session(state.db_engine) as session: - detail = db_service.get_stratum_detail(session, stratum_id) - - if detail is None: - raise HTTPException( - status_code=404, detail=f"Stratum {stratum_id} not found" - ) - return StratumDetail(**detail) diff --git a/backend/routes/summary.py b/backend/routes/summary.py deleted file mode 100644 index ba66a2c..0000000 --- a/backend/routes/summary.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Run-level summary scorecard for the landing page.""" - -from typing import Annotated - -import numpy as np -from fastapi import APIRouter, Depends, Query - -from backend.state import AppState, get_state - -router = APIRouter() - - -@router.get("/summary") -def get_summary( - error_bins: Annotated[int, Query(ge=4, le=60)] = 20, - worst_n: Annotated[int, Query(ge=1, le=50)] = 10, - state: AppState = Depends(get_state), -): - """Headline scorecard for a single run. - - Returns headline metrics, an abs-rel-error histogram, worst-fit categories - (grouped by variable), worst-fit targets, and weight-health stats. Designed - to be the only call needed to render the landing page. - """ - df = state.targets_enriched - if df.empty or "abs_rel_error" not in df.columns: - return _empty_summary(state) - if "included" in df.columns: - included = df[df["included"].astype(bool)] - else: - included = df - - abs_err_all = included["abs_rel_error"].to_numpy(dtype=float) - rel_err_all = included["rel_error"].to_numpy(dtype=float) - finite_mask = np.isfinite(abs_err_all) & np.isfinite(rel_err_all) - abs_err = abs_err_all[finite_mask] - rel_err = rel_err_all[finite_mask] - - # n_targets_with_estimate is computability coverage — counted across the - # WHOLE bundle, not just the included subset. In sandbox mode every X row - # produces an estimate (so this equals n_targets); in dataset mode only - # the MVP-evaluable subset does. - full_abs = df["abs_rel_error"].to_numpy(dtype=float) - n_with_estimate = int(np.sum(np.isfinite(full_abs))) - - headline = { - "dataset_id": state.dataset_id, - "run_id": state.run_id, - "n_targets": int(len(df)), - "n_targets_included": int(len(included)), - "n_targets_with_estimate": n_with_estimate, - "median_abs_rel_error": _safe_float(np.median(abs_err)) if len(abs_err) else None, - "mean_abs_rel_error": _safe_float(np.mean(abs_err)) if len(abs_err) else None, - "p95_abs_rel_error": _safe_float(np.percentile(abs_err, 95)) if len(abs_err) else None, - "pct_within_5pct": _pct(abs_err, 0.05), - "pct_within_10pct": _pct(abs_err, 0.10), - "pct_within_25pct": _pct(abs_err, 0.25), - "total_loss": _safe_float(np.sum(rel_err ** 2)), - "n_households": int(state.n_households), - "time_period": int(state.time_period), - } - - error_distribution = _histogram(abs_err, error_bins) - - worst_by_variable = _group_summary(included, "variable", top=worst_n) - worst_by_geo_level = _group_summary(included, "geo_level", top=worst_n) - - # Rank worst targets by loss_contribution when we have it (pkl mode); - # otherwise fall back to abs_rel_error so we still show meaningful rows - # in dataset mode where loss_contribution is uniformly 0. - rank_col = "loss_contribution" - if included[rank_col].fillna(0).max() == 0: - rank_col = "abs_rel_error" - worst_targets = ( - included.nlargest(worst_n, rank_col)[ - [ - "target_name", "variable", "geo_level", "value", - "estimate", "rel_error", "abs_rel_error", "loss_contribution", - ] - ] - .assign(target_idx=lambda d: d.index.astype(int)) - .to_dict(orient="records") - ) - weight_health = _weight_health(state) - - payload = { - "headline": headline, - "error_distribution": error_distribution, - "worst_by_variable": worst_by_variable, - "worst_by_geo_level": worst_by_geo_level, - "worst_targets": worst_targets, - "weight_health": weight_health, - } - return _scrub_nans(payload) - - -def _scrub_nans(obj): - """Recursively replace non-finite floats with None so JSON serialisation - doesn't choke. Dataset mode produces NaN estimates for any target that - needs entity-mapped constraint evaluation.""" - if isinstance(obj, dict): - return {k: _scrub_nans(v) for k, v in obj.items()} - if isinstance(obj, list): - return [_scrub_nans(x) for x in obj] - if isinstance(obj, float) and not np.isfinite(obj): - return None - return obj - - -def _empty_summary(state: AppState) -> dict: - return { - "headline": { - "dataset_id": state.dataset_id, - "run_id": state.run_id, - "n_targets": 0, - "n_targets_included": 0, - "median_abs_rel_error": None, - "mean_abs_rel_error": None, - "p95_abs_rel_error": None, - "pct_within_5pct": None, - "pct_within_10pct": None, - "pct_within_25pct": None, - "total_loss": 0.0, - "n_households": int(state.n_households), - "time_period": int(state.time_period), - }, - "error_distribution": [], - "worst_by_variable": [], - "worst_by_geo_level": [], - "worst_targets": [], - "weight_health": _weight_health(state), - } - - -def _safe_float(x) -> float | None: - try: - v = float(x) - except (TypeError, ValueError): - return None - if not np.isfinite(v): - return None - return v - - -def _pct(arr: np.ndarray, threshold: float) -> float | None: - if len(arr) == 0: - return None - return float(np.mean(arr <= threshold)) - - -def _histogram(arr: np.ndarray, bins: int, cap: float = 2.0) -> list[dict]: - """Equal-width bins over [0, cap] plus a single overflow bucket. - - With default cap=2.0 and bins=20, each bucket is 10% wide; anything ≥200% - abs_rel_error lands in the overflow bucket. This keeps the chart readable - even when a handful of targets have pathological errors (e.g. 157,675%). - """ - if len(arr) == 0: - return [] - in_range = arr[arr <= cap] - counts, edges = np.histogram(in_range, bins=bins, range=(0.0, cap)) - overflow = int(np.sum(arr > cap)) - out = [ - {"bin_min": float(edges[i]), "bin_max": float(edges[i + 1]), - "count": int(counts[i])} - for i in range(len(counts)) - ] - if overflow: - out.append({"bin_min": cap, "bin_max": float(np.max(arr)), - "count": overflow, "overflow": True}) - return out - - -def _group_summary(df, by: str, top: int) -> list[dict]: - if by not in df.columns: - return [] - grouped = df.groupby(by, dropna=False).agg( - n_targets=("abs_rel_error", "size"), - mean_abs_rel_error=("abs_rel_error", "mean"), - median_abs_rel_error=("abs_rel_error", "median"), - total_loss=("loss_contribution", "sum"), - ) - sort_col = "total_loss" - if grouped["total_loss"].fillna(0).eq(0).all(): - sort_col = "mean_abs_rel_error" - grouped = grouped.sort_values(sort_col, ascending=False).head(top) - out = [] - for key, row in grouped.iterrows(): - out.append({ - "group": str(key) if key is not None else "(none)", - "n_targets": int(row["n_targets"]), - "mean_abs_rel_error": _safe_float(row["mean_abs_rel_error"]), - "median_abs_rel_error": _safe_float(row["median_abs_rel_error"]), - "total_loss": _safe_float(row["total_loss"]), - }) - return out - - -def _weight_health(state: AppState) -> dict: - g = state.g_weights - finals = state.final_weights - return { - "n_households": int(len(g)), - "pct_zero_g": float(np.mean(g == 0)) if len(g) else None, - "pct_negative_final": float(np.mean(finals < 0)) if len(finals) else None, - "pct_extreme_g_high": float(np.mean(g > 10)) if len(g) else None, - "pct_extreme_g_low": float(np.mean((g > 0) & (g < 0.1))) if len(g) else None, - "g_median": _safe_float(np.median(g)) if len(g) else None, - "g_p95": _safe_float(np.percentile(g, 95)) if len(g) else None, - "g_p5": _safe_float(np.percentile(g, 5)) if len(g) else None, - } diff --git a/backend/routes/target_inventory.py b/backend/routes/target_inventory.py deleted file mode 100644 index f97ca9b..0000000 --- a/backend/routes/target_inventory.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Endpoints for the cross-tier Target Inventory view. - -Serves the per-tier audit summary (small, committed) and the full per-record -union (large, gitignored, regenerable). The latter is streamed/paginated so -the frontend can render filtered slices without shipping the 25MB blob. -""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -import pandas as pd -from fastapi import APIRouter, Depends, HTTPException, Query - -from backend.state import AppState, get_state - -router = APIRouter() - - -_GEO_VARS = {"state_fips", "congressional_district_geoid", "ucgid_str"} - - -def _norm_geo_id(val) -> str: - """Mirror schema._norm_geo_id so signatures match across tiers.""" - if val is None or val == "" or pd.isna(val): - return "" - s = str(val) - try: - f = float(s) - except (TypeError, ValueError): - return s - import math - if not math.isfinite(f): - return s - return str(int(f)) - - -def _signature_for_enriched_row(row, target_name_constraints=None) -> tuple: - """Mirror TargetRecord.signature() so loaded-run targets sig-match the - inventory rows. Pulls constraints from the row's parsed target_name when - needed, mirroring the same logic used by the DB-load path.""" - variable = str(row.get("variable", "")) - geo_level = (row.get("geo_level") or "national") - geo_id = _norm_geo_id(row.get("geographic_id")) - period = int(row.get("period") or 0) - # Reuse already-parsed constraints if provided; else infer empty. - cons = tuple(sorted( - (c[0], c[1], _norm_geo_id(c[2])) - for c in (target_name_constraints or []) - )) - return (variable, geo_level, geo_id, period, cons) - -DATA_ROOT = Path(__file__).resolve().parents[1] / "data" -SUMMARY_PATH = DATA_ROOT / "target_index.json" -UNION_PATH = DATA_ROOT / "target_index.json.union.jsonl" - - -def _load_summary() -> dict: - if not SUMMARY_PATH.exists(): - raise HTTPException( - status_code=503, - detail=( - "target_index.json not built. Run " - "`python -m backend.scripts.target_index.audit`." - ), - ) - return json.loads(SUMMARY_PATH.read_text()) - - -def _load_union() -> list[dict]: - if not UNION_PATH.exists(): - raise HTTPException( - status_code=503, - detail=( - "Per-record union not built. Run " - "`python -m backend.scripts.target_index.audit` to regenerate " - f"{UNION_PATH.name} (gitignored, ~25MB)." - ), - ) - with UNION_PATH.open() as f: - return [json.loads(line) for line in f if line.strip()] - - -@router.get("/target-inventory/summary") -def get_summary(): - """The audit summary: per-tier counts + match rates.""" - return _load_summary() - - -def _build_estimate_lookup(state: AppState) -> dict[int, dict]: - """Map target_id → {estimate, rel_error, target_idx} from the loaded run. - - The DB-tier inventory rows store the DB target_id in their source_row - ("target_id=N/stratum_id=M"), so joining by target_id is reliable — - no fragile signature reconstruction needed. - """ - df = state.targets_enriched - if df is None or df.empty or "target_id" not in df.columns: - return {} - lookup: dict[int, dict] = {} - for idx, row in df.iterrows(): - tid = row.get("target_id") - if pd.isna(tid): - continue - est = row.get("estimate") - rel_err = row.get("rel_error") - lookup[int(tid)] = { - "estimate": float(est) if pd.notna(est) else None, - "rel_error": float(rel_err) if pd.notna(rel_err) else None, - "target_idx": int(idx), - } - return lookup - - -_TARGET_ID_RE = re.compile(r"target_id=(\d+)") - - -def _target_id_from_source_row(source_row: str) -> int | None: - """Inventory rows from the DB tier have source_row = 'target_id=N/stratum_id=M'. - Extract N for the join.""" - if not source_row: - return None - m = _TARGET_ID_RE.search(source_row) - return int(m.group(1)) if m else None - - -@router.get("/target-inventory") -def list_targets( - tier: str | None = Query(None, description="Filter by storage tier (db/csv/python)"), - source_path: str | None = Query(None, description="Filter by exact source file path"), - variable: str | None = Query(None, description="Filter by PE variable name"), - in_db: bool | None = Query(None, description="Filter to only matched (true) or only unmatched (false)"), - search: str | None = Query(None, description="Substring match against variable/source/notes"), - limit: int = Query(100, ge=1, le=2000), - offset: int = Query(0, ge=0), - state: AppState = Depends(get_state), -): - """Paginated listing of the full union. Each row already carries - storage_tier / source_path / source_row / notes / signature, plus an - `in_db` flag derived from cross-tier matching against policy_data.db.""" - union = _load_union() - - def _freeze(x): - """Recursively convert nested lists to tuples so the signature is - hashable (the JSONL stores constraints as list-of-lists).""" - if isinstance(x, list): - return tuple(_freeze(i) for i in x) - return x - - # Build a set of DB signatures for the in_db flag. - db_sigs = {_freeze(r["signature"]) for r in union if r.get("storage_tier") == "db"} - - # Build target_id → {estimate, rel_error, target_idx} from the loaded run. - # DB-tier inventory rows carry target_id in their source_row; that's the - # cleanest join key. For non-DB rows we'd need a more elaborate mapping; - # those stay '—' here (the audit page already shows the gap). - estimate_lookup = _build_estimate_lookup(state) - - # MVP evaluator cache (lazy: only built if there's a non-DB row that - # needs computation). - eval_cache_box: list = [None] - - def _get_eval_cache(): - if eval_cache_box[0] is not None: - return eval_cache_box[0] - if state.sim_service is None: - return None - from backend.services.stratum_evaluator import EvalCache - eval_cache_box[0] = EvalCache(state) - return eval_cache_box[0] - - def annotate(r): - sig = _freeze(r["signature"]) - # DB rows trivially have in_db=True; otherwise check membership. - r["in_db"] = r.get("storage_tier") == "db" or sig in db_sigs - - r["estimate"] = None - r["rel_error"] = None - r["target_idx"] = None - r["eval_note"] = "" - - if r.get("storage_tier") == "db": - # Cheap path: pull the loaded run's estimate by target_id. - tid = _target_id_from_source_row(r.get("source_row") or "") - if tid is not None and tid in estimate_lookup: - hit = estimate_lookup[tid] - r["estimate"] = hit["estimate"] - r["rel_error"] = hit["rel_error"] - r["target_idx"] = hit["target_idx"] - r["eval_note"] = "from loaded calibration X·w" - else: - # Authored-only row: try the MVP evaluator. Geographic-only and - # simple-constraint cases will return a real number. - cache = _get_eval_cache() - if cache is not None: - from backend.services.stratum_evaluator import evaluate_signature - cons = r.get("constraints") or [] - try: - est, note = evaluate_signature( - variable=r["variable"], - geo_level=r.get("geo_level"), - geographic_id=r.get("geographic_id"), - constraints=[tuple(c) for c in cons], - is_count=bool(r.get("is_count")), - cache=cache, - period=r.get("period"), - ) - r["estimate"] = est - r["eval_note"] = note - target = r.get("value") - if est is not None and target not in (None, 0): - r["rel_error"] = (est - target) / abs(target) - except Exception as exc: - r["eval_note"] = f"evaluator error: {exc}" - return r - - rows = [annotate(r) for r in union] - - # Filters - if tier: - rows = [r for r in rows if r.get("storage_tier") == tier] - if source_path: - rows = [r for r in rows if r.get("source_path") == source_path] - if variable: - rows = [r for r in rows if r.get("variable") == variable] - if in_db is not None: - rows = [r for r in rows if r.get("in_db") is in_db] - if search: - s = search.lower() - rows = [ - r for r in rows - if s in (r.get("variable") or "").lower() - or s in (r.get("source_path") or "").lower() - or s in (r.get("notes") or "").lower() - ] - - total = len(rows) - return { - "items": rows[offset : offset + limit], - "total": total, - "offset": offset, - "limit": limit, - } diff --git a/backend/routes/targets.py b/backend/routes/targets.py deleted file mode 100644 index 94360ea..0000000 --- a/backend/routes/targets.py +++ /dev/null @@ -1,857 +0,0 @@ -"""Target analysis endpoints.""" - -import operator as op_module -from typing import Annotated - -import numpy as np -import pandas as pd -from fastapi import APIRouter, Depends, HTTPException, Query, Request -from sqlmodel import Session - -from backend.services.geo_utils import geo_display_name -from backend.state import get_state -from backend.models import ( - ConstraintCheck, - ConstraintDiffResponse, - ContributorRow, - ConvergencePoint, - EligibilityAuditResponse, - ErrorDecomposition, - ProvenanceResponse, - TargetListResponse, - TargetRow, -) -from backend.services import db_service, matrix_ops -from backend.state import AppState - -router = APIRouter() - -_OPS = { - "gt": op_module.gt, - "gte": op_module.ge, - "lt": op_module.lt, - "lte": op_module.le, - "eq": op_module.eq, - "ne": op_module.ne, -} - - -def _validate_target_idx(state: AppState, target_idx: int) -> None: - if target_idx < 0 or target_idx >= state.n_targets: - raise HTTPException( - status_code=404, - detail=f"target_idx {target_idx} out of range [0, {state.n_targets})", - ) - - -def _require_matrix_artifacts(state: AppState) -> None: - if state.X_csr.shape[0] == 0 or state.X_csr.shape[1] == 0 or state.X_csr.nnz == 0: - raise HTTPException( - status_code=501, - detail=( - "This run does not publish the calibration sparse matrix. " - "Target detail tabs that need household-level contributors " - "are unavailable for dataset-mode runs." - ), - ) - - -# --- Literal-segment routes BEFORE parametric routes --- - - -@router.get("/search") -def search_targets( - variable: str, - sort_by: str = "abs_rel_error", - state: AppState = Depends(get_state), -) -> list[dict]: - if state.db_engine is None: - raise HTTPException(status_code=503, detail="No database connected") - - with Session(state.db_engine) as session: - db_results = db_service.search_targets(session, variable) - - enriched = state.targets_enriched - out = [] - for r in db_results: - match = enriched[enriched.get("target_id") == r["target_id"]] - error_info = {} - if len(match) > 0: - row = match.iloc[0] - error_info = { - "target_idx": int(match.index[0]), - "estimate": float(row.get("estimate", 0)), - "rel_error": float(row.get("rel_error", 0)), - "abs_rel_error": float(row.get("abs_rel_error", 0)), - "loss_contribution": float(row.get("loss_contribution", 0)), - "n_contributors": int(row.get("n_contributors", 0)), - } - out.append({**r, **error_info}) - - if sort_by in ("abs_rel_error", "loss_contribution", "rel_error"): - out.sort(key=lambda x: abs(x.get(sort_by, 0)), reverse=True) - return out - - -@router.get("/worst-fit") -def worst_fit( - limit: int = 20, - included_only: bool = True, - state: AppState = Depends(get_state), -) -> list[TargetRow]: - enriched = state.targets_enriched - if included_only: - enriched = enriched[enriched["included"]] - enriched = enriched.sort_values( - "abs_rel_error", ascending=False - ).head(limit) - return [_target_row(enriched, idx) for idx in enriched.index] - - -# Error buckets — abs_rel_error ranges. Order matters for display. -ERROR_BUCKETS = { - "excellent": (0.0, 0.05), - "good": (0.05, 0.20), - "poor": (0.20, 0.50), - "extreme": (0.50, float("inf")), -} - - -def _available_bundles_for_state(state) -> "frozenset[str] | None": - """Look up which bundle h5s the loaded run actually publishes on HF. - - Returns ``None`` for runs without a resolvable HF repo (pkl-mode - sandbox); callers that get ``None`` should treat the canonical - bundle mapping as a best-effort label rather than a verified fact. - """ - try: - from backend.services.runs import get_dataset - from backend.services.bundle_availability import published_bundles - ds = get_dataset(state.dataset_id) - except Exception: - return None - if ds is None or getattr(ds, "layout", None) not in { - "staging", "root", "staging-root", - }: - return None - return published_bundles(ds.repo_id, state.run_id) - - -def _apply_target_filters( - df, - *, - search: str | None = None, - variables: list[str] | None = None, - geo_levels: list[str] | None = None, - error_buckets: list[str] | None = None, - sources: list[str] | None = None, - geographic_id: str | None = None, - state_fips: list[int] | None = None, - domain_variable: str | None = None, - min_abs_rel_error: float | None = None, - included_only: bool | None = None, - dataset_files: list[str] | None = None, - available_bundles: "frozenset[str] | None" = None, -): - """Apply the standard filter set used by list_targets and facets.""" - if included_only is not None: - df = df[df["included"] == included_only] - if variables: - df = df[df["variable"].isin(variables)] - if geo_levels: - df = df[df["geo_level"].isin(geo_levels)] - if error_buckets: - # Strict: if user passed bucket names but none are recognised, return - # an empty result rather than silently ignoring the filter. - masks = [] - for bucket in error_buckets: - if bucket not in ERROR_BUCKETS: - continue - lo, hi = ERROR_BUCKETS[bucket] - masks.append((df["abs_rel_error"] >= lo) & (df["abs_rel_error"] < hi)) - if not masks: - df = df.iloc[0:0] - else: - combined = masks[0] - for m in masks[1:]: - combined = combined | m - df = df[combined] - if sources and "source" in df.columns: - df = df[df["source"].isin(sources)] - if dataset_files: - # Each target maps to one calibrated h5 in us-data's pipeline. If - # `available_bundles` is set, fall back to the federal bundle when - # the conventional per-bundle h5 doesn't exist for this run. - from backend.services.geo_utils import runtime_dataset_bundle_for - wanted = set(dataset_files) - df = df[df.apply( - lambda r: runtime_dataset_bundle_for( - r.get("geo_level"), r.get("geographic_id"), - available=available_bundles, - ) in wanted, - axis=1, - )] - if geographic_id: - df = df[df["geographic_id"].astype(str) == str(geographic_id)] - if state_fips: - fips_set = set(state_fips) - def _matches_any_state(gid): - s = str(gid) - if not s.isdigit(): - return s in {str(f) for f in fips_set} - try: - return int(s) in fips_set or (int(s) // 100) in fips_set - except ValueError: - return False - df = df[df["geographic_id"].apply(_matches_any_state)] - if domain_variable: - df = df[ - df["domain_variable"].str.contains( - domain_variable, case=False, na=False - ) - ] - if min_abs_rel_error is not None: - df = df[df["abs_rel_error"] >= min_abs_rel_error] - if search: - # Search across target_name + variable + domain (case-insensitive) - s = search.lower() - haystack = ( - df["target_name"].fillna("").astype(str).str.lower() - + " " - + df["variable"].fillna("").astype(str).str.lower() - + " " - + df.get("domain", df.get("domain_variable", "")).fillna("").astype(str).str.lower() - ) - df = df[haystack.str.contains(s, regex=False, na=False)] - return df - - -@router.get("") -def list_targets( - request: Request, - sort_by: str = "loss_contribution", - sort_order: str = "desc", - search: str | None = None, - variable: Annotated[list[str] | None, Query()] = None, - geo_level: Annotated[list[str] | None, Query()] = None, - error_bucket: Annotated[list[str] | None, Query()] = None, - source: Annotated[list[str] | None, Query()] = None, - geographic_id: str | None = None, - state_fips: Annotated[list[int] | None, Query(alias="state_fips")] = None, - domain_variable: str | None = None, - min_abs_rel_error: float | None = None, - included_only: bool | None = None, - compare_run: str | None = Query( - None, description="Second run id (same dataset) to join for compare mode." - ), - dataset_file: Annotated[list[str] | None, Query()] = None, - limit: int = 50, - offset: int = 0, - state: AppState = Depends(get_state), -) -> TargetListResponse: - available_bundles = _available_bundles_for_state(state) - df = _apply_target_filters( - state.targets_enriched, - search=search, - variables=variable, - geo_levels=geo_level, - error_buckets=error_bucket, - sources=source, - geographic_id=geographic_id, - state_fips=state_fips, - domain_variable=domain_variable, - min_abs_rel_error=min_abs_rel_error, - included_only=included_only, - dataset_files=dataset_file, - available_bundles=available_bundles, - ) - - # Per-bundle re-evaluation. Kicks in when the caller filtered to - # exactly one published bundle other than the federal one we - # already loaded — then estimates come from that bundle's own h5. - bundle_evaluated: str | None = None - if ( - dataset_file - and len(dataset_file) == 1 - and available_bundles - and dataset_file[0] in available_bundles - and dataset_file[0] != "enhanced_cps_2024.h5" - and not df.empty - ): - only = dataset_file[0] - try: - from backend.services.runs import get_dataset - from backend.services.bundle_eval import evaluate_bundle - ds = get_dataset(state.dataset_id) - df = evaluate_bundle( - df, - repo_id=ds.repo_id, - run_id=state.run_id, - bundle=only, - time_period=state.time_period, - ) - bundle_evaluated = only - except Exception as exc: - import logging as _logging - _logging.getLogger(__name__).warning( - "Per-bundle eval for %s failed (%s); using federal-fit numbers.", - only, exc, - ) - - # Compare-run join: enriches each row with estimate_b / rel_error_b / - # abs_rel_error_b / delta from the second run, joined on target_id. - # Loaded through the registry so a cold compare-run picks up the same - # entity-aware evaluator pass. - df_b = None - if request is not None and compare_run and compare_run != state.run_id: - try: - state_b = request.app.state.registry.get(state.dataset_id, compare_run) - df_b = state_b.targets_enriched - except Exception: - df_b = None - if df_b is not None and not df_b.empty: - b_cols = df_b[["target_id", "estimate", "rel_error", "abs_rel_error"]].rename( - columns={ - "estimate": "estimate_b", - "rel_error": "rel_error_b", - "abs_rel_error": "abs_rel_error_b", - } - ) - df = df.merge(b_cols, on="target_id", how="left") - df["delta"] = df["abs_rel_error_b"] - df["abs_rel_error"] - - total = len(df) - ascending = sort_order == "asc" - if sort_by in df.columns: - effective_sort = sort_by - if sort_by == "loss_contribution": - losses = df["loss_contribution"].to_numpy(dtype=float) - if len(losses) == 0 or not np.any(np.nan_to_num(losses) != 0): - effective_sort = "abs_rel_error" - df = df.sort_values(effective_sort, ascending=ascending) - df = df.iloc[offset : offset + limit] - - has_compare = df_b is not None and not df_b.empty - return TargetListResponse( - items=[_target_row(df, idx, with_compare=has_compare) for idx in df.index], - total=total, - offset=offset, - limit=limit, - bundle_evaluated=bundle_evaluated, - ) - - -@router.get("/facets") -def get_facets( - search: str | None = None, - variable: Annotated[list[str] | None, Query()] = None, - geo_level: Annotated[list[str] | None, Query()] = None, - error_bucket: Annotated[list[str] | None, Query()] = None, - source: Annotated[list[str] | None, Query()] = None, - included_only: bool | None = None, - state_fips: Annotated[list[int] | None, Query(alias="state_fips")] = None, - state: AppState = Depends(get_state), -): - """Per-facet counts. For each facet, counts are computed against the - other active filters (so a facet doesn't suppress its own selection).""" - - def _filtered(exclude: str): - return _apply_target_filters( - state.targets_enriched, - search=search, - variables=variable if exclude != "variable" else None, - geo_levels=geo_level if exclude != "geo_level" else None, - error_buckets=error_bucket if exclude != "error_bucket" else None, - sources=source if exclude != "source" else None, - included_only=included_only, - state_fips=state_fips, - ) - - def _value_counts_with_loss(df, col: str): - if col not in df.columns: - return [] - grouped = ( - df.groupby(col, dropna=False) - .agg(count=(col, "size"), - total_loss=("loss_contribution", "sum")) - .sort_values("total_loss", ascending=False) - ) - out = [] - for key, row in grouped.iterrows(): - out.append({ - "value": "(none)" if key is None or (isinstance(key, float) and key != key) else str(key), - "count": int(row["count"]), - "total_loss": float(row["total_loss"]), - }) - return out - - by_variable = _value_counts_with_loss(_filtered("variable"), "variable") - by_geo_level = _value_counts_with_loss(_filtered("geo_level"), "geo_level") - by_source = _value_counts_with_loss(_filtered("source"), "source") - - # Per-h5-bundle counts: which calibrated dataset each target rolls - # up into. Runtime-aware so we only list bundles the run actually - # publishes — for a federal-only GHA run that means a single entry - # holding all 40k targets, not the theoretical 200 names. - from backend.services.geo_utils import runtime_dataset_bundle_for - available_bundles = _available_bundles_for_state(state) - df_bundles = _filtered("dataset_file") - df_bundles = df_bundles.assign( - _bundle=df_bundles.apply( - lambda r: runtime_dataset_bundle_for( - r.get("geo_level"), r.get("geographic_id"), - available=available_bundles, - ), - axis=1, - ) - ) - bundle_counts = ( - df_bundles.groupby("_bundle", dropna=False) - .size() - .sort_values(ascending=False) - .head(200) - ) - by_dataset_file = [ - {"value": str(k), "count": int(v)} for k, v in bundle_counts.items() - ] - - # Error buckets — count distribution within the current selection-aware df. - df_for_buckets = _filtered("error_bucket") - abs_err = df_for_buckets["abs_rel_error"].to_numpy() if "abs_rel_error" in df_for_buckets.columns else [] - by_error_bucket = [] - for name, (lo, hi) in ERROR_BUCKETS.items(): - if len(abs_err) == 0: - count = 0 - else: - count = int(((abs_err >= lo) & (abs_err < hi)).sum()) - by_error_bucket.append({"value": name, "count": count}) - - # Status: included vs skipped - df_status = _apply_target_filters( - state.targets_enriched, - search=search, - variables=variable, - geo_levels=geo_level, - error_buckets=error_bucket, - state_fips=state_fips, - ) - by_status = [] - if "included" in df_status.columns: - included_ct = int(df_status["included"].sum()) - skipped_ct = int((~df_status["included"]).sum()) - by_status = [ - {"value": "included", "count": included_ct}, - {"value": "skipped", "count": skipped_ct}, - ] - - return { - "by_variable": by_variable, - "by_geo_level": by_geo_level, - "by_source": by_source, - "by_error_bucket": by_error_bucket, - "by_status": by_status, - "by_dataset_file": by_dataset_file, - "buckets_definition": { - k: {"min": v[0], "max": None if v[1] == float("inf") else v[1]} - for k, v in ERROR_BUCKETS.items() - }, - } - - -@router.get("/source-summary") -def get_source_summary( - included_only: bool | None = True, - state: AppState = Depends(get_state), -): - """Per-source stats: count, mean/median |rel_error|, % within ±10%. - Useful for spotting upstream sources the calibration fits well vs poorly. - """ - df = state.targets_enriched - if df.empty or "source" not in df.columns: - return {"sources": []} - if included_only and "included" in df.columns: - df = df[df["included"].astype(bool)] - - grouped = ( - df.dropna(subset=["source"]) - .groupby("source", dropna=False) - .agg( - n_targets=("source", "size"), - mean_abs_rel_error=("abs_rel_error", "mean"), - median_abs_rel_error=("abs_rel_error", "median"), - total_loss=("loss_contribution", "sum"), - ) - .reset_index() - .sort_values("n_targets", ascending=False) - ) - out = [] - for _, r in grouped.iterrows(): - # Pct within 10% - sub = df[df["source"] == r["source"]] - abs_err = sub["abs_rel_error"].to_numpy() - finite = abs_err[np.isfinite(abs_err)] - within_10 = float((finite <= 0.10).mean()) if len(finite) else None - out.append({ - "source": str(r["source"]), - "n_targets": int(r["n_targets"]), - "mean_abs_rel_error": _safe_float(r["mean_abs_rel_error"]), - "median_abs_rel_error": _safe_float(r["median_abs_rel_error"]), - "total_loss": _safe_float(r["total_loss"]), - "pct_within_10pct": within_10, - }) - return {"sources": out} - - -# --- Parametric routes --- - - -@router.get("/{target_idx}/error-decomposition") -def error_decomposition( - target_idx: int, - state: AppState = Depends(get_state), -) -> ErrorDecomposition: - _validate_target_idx(state, target_idx) - _require_matrix_artifacts(state) - target_value = float(state.targets_enriched.iloc[target_idx]["value"]) - decomp = matrix_ops.compute_error_decomposition( - state.X_csr, target_idx, target_value, - state.initial_weights, state.final_weights, - ) - concentration = matrix_ops.compute_concentration( - state.X_csr, target_idx, state.final_weights, - ) - return ErrorDecomposition( - target_name=state.target_names[target_idx], - concentration=concentration, - **decomp, - ) - - -@router.get("/{target_idx}/provenance") -def provenance( - target_idx: int, - state: AppState = Depends(get_state), -) -> ProvenanceResponse: - _validate_target_idx(state, target_idx) - if state.db_engine is None: - raise HTTPException(status_code=503, detail="No database connected") - - row = state.targets_enriched.iloc[target_idx] - target_id = row.get("target_id") - if target_id is None: - raise HTTPException(status_code=404, detail="No target_id mapping") - - with Session(state.db_engine) as session: - prov = db_service.get_target_provenance(session, int(target_id)) - - if prov is None: - raise HTTPException(status_code=404, detail="Target not found in database") - - prov["geo_level"] = row.get("geo_level") - prov["geographic_id"] = str(row.get("geographic_id", "")) - prov["uprating_factor"] = row.get("uprating_factor") - if prov["uprating_factor"] and prov["value"]: - prov["uprated_value"] = prov["value"] * prov["uprating_factor"] - else: - prov["uprated_value"] = prov["value"] - - return ProvenanceResponse(**prov) - - -@router.get("/{target_idx}/eligibility-audit") -def eligibility_audit( - target_idx: int, - criterion_variable: str = Query(...), - criterion_operator: str = Query("gt"), - criterion_value: float = Query(0), - state: AppState = Depends(get_state), -) -> EligibilityAuditResponse: - _validate_target_idx(state, target_idx) - _require_matrix_artifacts(state) - if criterion_operator not in _OPS: - raise HTTPException(status_code=400, detail=f"Invalid operator: {criterion_operator}") - if not state.sim_service.variable_exists(criterion_variable): - raise HTTPException(status_code=400, detail=f"Unknown variable: {criterion_variable}") - - contributors = matrix_ops.get_target_contributors(state.X_csr, target_idx) - criterion_vals = state.sim_service.calculate(criterion_variable, map_to="household") - contributor_vals = criterion_vals[contributors] - - op_fn = _OPS[criterion_operator] - meets = op_fn(contributor_vals, criterion_value) - fails = ~meets - - n_meet = int(meets.sum()) - n_fail = int(fails.sum()) - total = len(contributors) - - # Weighted contribution from failing households - row = state.X_csr[target_idx, :] - fail_cols = contributors[fails] - if len(fail_cols) > 0: - fail_weighted = float( - row[:, fail_cols].multiply(state.final_weights[fail_cols]).sum() - ) - else: - fail_weighted = 0.0 - - estimate = float(state.targets_enriched.iloc[target_idx]["estimate"]) - pct_est_failing = (fail_weighted / estimate * 100) if estimate != 0 else 0.0 - - return EligibilityAuditResponse( - target_name=state.target_names[target_idx], - total_contributors=total, - meet_criterion=n_meet, - fail_criterion=n_fail, - pct_failing=n_fail / total * 100 if total > 0 else 0.0, - weighted_contribution_from_failing=fail_weighted, - pct_estimate_from_failing=pct_est_failing, - diagnosis=( - f"{n_fail / total:.1%} of contributors do not meet criterion " - f"({criterion_variable} {criterion_operator} {criterion_value}). " - f"They contribute {pct_est_failing:.1f}% of the estimate." - if total > 0 - else "No contributors" - ), - ) - - -@router.get("/{target_idx}/constraint-diff") -def constraint_diff( - target_idx: int, - state: AppState = Depends(get_state), -) -> ConstraintDiffResponse: - _validate_target_idx(state, target_idx) - _require_matrix_artifacts(state) - if state.db_engine is None: - raise HTTPException(status_code=503, detail="No database connected") - - row = state.targets_enriched.iloc[target_idx] - stratum_id = row.get("stratum_id") - if stratum_id is None: - raise HTTPException(status_code=404, detail="No stratum_id mapping") - - with Session(state.db_engine) as session: - constraints = db_service.get_target_constraints( - session, int(stratum_id) - ) - - contributors = matrix_ops.get_target_contributors(state.X_csr, target_idx) - geo_vars = {"state_fips", "congressional_district_geoid", "ucgid_str"} - checks = [] - - for c in constraints: - if c["variable"] in geo_vars: - continue - if not state.sim_service.variable_exists(c["variable"]): - checks.append(ConstraintCheck( - variable=c["variable"], - operation=c["operation"], - value=c["value"], - contributors_satisfying=0, - contributors_violating=0, - pct_violating=0, - status="SKIPPED_UNKNOWN_VARIABLE", - )) - continue - - vals = state.sim_service.calculate(c["variable"], map_to="household") - contributor_vals = vals[contributors] - - threshold = _parse_value(c["value"]) - op_map = { - ">": op_module.gt, - ">=": op_module.ge, - "<": op_module.lt, - "<=": op_module.le, - "==": op_module.eq, - "!=": op_module.ne, - } - op_fn = op_map.get(c["operation"]) - if op_fn is None: - continue - - meets = op_fn(contributor_vals, threshold) - n_meet = int(meets.sum()) - n_fail = len(contributors) - n_meet - pct_fail = n_fail / len(contributors) * 100 if len(contributors) > 0 else 0 - - if pct_fail > 10: - status = "VIOLATION" - elif pct_fail > 1: - status = "MINOR_VIOLATION" - else: - status = "OK" - - checks.append(ConstraintCheck( - variable=c["variable"], - operation=c["operation"], - value=c["value"], - contributors_satisfying=n_meet, - contributors_violating=n_fail, - pct_violating=pct_fail, - status=status, - )) - - return ConstraintDiffResponse( - target_name=state.target_names[target_idx], - stratum_id=int(stratum_id), - constraints=checks, - ) - - -@router.get("/{target_idx}/contributors") -def contributors( - target_idx: int, - min_g_weight: float | None = None, - poverty_only: bool = False, - sort_by: str = "g_weight", - limit: int = 50, - offset: int = 0, - state: AppState = Depends(get_state), -) -> list[ContributorRow]: - _validate_target_idx(state, target_idx) - _require_matrix_artifacts(state) - contribs = matrix_ops.get_target_contributions( - state.X_csr, target_idx, state.final_weights, - ) - hh = state.households_df - merged = contribs.merge( - hh[["household_idx", "income", "g_weight", "in_poverty", "state"]], - on="household_idx", - ) - - if poverty_only: - merged = merged[merged["in_poverty"]] - if min_g_weight is not None: - merged = merged[merged["g_weight"] >= min_g_weight] - - if sort_by in merged.columns: - merged = merged.sort_values(sort_by, ascending=False) - merged = merged.iloc[offset : offset + limit] - - return [ - ContributorRow( - household_idx=int(r["household_idx"]), - raw_value=float(r["raw_value"]), - weighted_value=float(r["weighted_value"]), - income=float(r["income"]), - g_weight=float(r["g_weight"]), - in_poverty=bool(r["in_poverty"]), - state=int(r["state"]), - ) - for _, r in merged.iterrows() - ] - - -@router.get("/{target_idx}/convergence") -def convergence( - target_idx: int, - state: AppState = Depends(get_state), -) -> list[ConvergencePoint]: - _validate_target_idx(state, target_idx) - if state.cal_log is None: - raise HTTPException(status_code=404, detail="No calibration log available") - - name = state.target_names[target_idx] - filtered = state.cal_log[state.cal_log["target_name"] == name] - return [ - ConvergencePoint( - epoch=int(r["epoch"]), - estimate=float(r["estimate"]), - target=float(r["target"]), - rel_error=float(r["rel_error"]), - loss=float(r["loss"]), - ) - for _, r in filtered.iterrows() - ] - - -# --- Helpers --- - - -def _nan_to_none(val): - """Convert pandas NaN to None for Pydantic compatibility.""" - if isinstance(val, float) and pd.isna(val): - return None - return val - - -def _safe_int(val) -> int | None: - if val is None or (isinstance(val, float) and pd.isna(val)): - return None - try: - return int(val) - except (TypeError, ValueError): - return None - - -def _safe_float(val) -> float | None: - if val is None or pd.isna(val): - return None - try: - out = float(val) - except (TypeError, ValueError): - return None - if not np.isfinite(out): - return None - return out - - -def _parse_value(val): - try: - return float(val) - except (TypeError, ValueError): - return val - - -def _parse_constraints_from_name(target_name: str) -> list[str]: - """Extract non-geographic constraints from target_name. - - Target names look like: national/snap/[tax_unit_is_filer==1,snap>0] - or: cd_0622/person_count/[age>=18,age<25] - or: national/rent (no brackets = no constraints) - """ - if not isinstance(target_name, str) or "[" not in target_name: - return [] - bracket = target_name[target_name.index("[") + 1 : target_name.rindex("]")] - if not bracket: - return [] - return [c.strip() for c in bracket.split(",") if c.strip()] - - -def _target_row(df, idx: int, with_compare: bool = False) -> TargetRow: - r = df.loc[idx] - gl = _nan_to_none(r.get("geo_level")) or "national" - gid = str(_nan_to_none(r.get("geographic_id")) or "US") - constraints = _parse_constraints_from_name(str(r.get("target_name", ""))) - target_value = _safe_float(r.get("value")) - estimate = _safe_float(r.get("estimate")) - abs_error = ( - abs(estimate - target_value) - if estimate is not None and target_value is not None - else None - ) - return TargetRow( - target_idx=idx, - target_id=_safe_int(r.get("target_id")), - variable=str(r.get("variable", "")), - geo_level=gl, - geographic_id=gid, - geo_display_name=geo_display_name(gl, gid), - constraints=constraints, - target_value=target_value, - estimate=estimate, - rel_error=_safe_float(r.get("rel_error")), - abs_error=abs_error, - loss_contribution=_safe_float(r.get("loss_contribution")), - included=bool(r.get("included", True)), - source=_nan_to_none(r.get("source")), - period=_safe_int(r.get("period")), - tolerance=_safe_float(r.get("tolerance")), - notes=_nan_to_none(r.get("notes")), - estimate_b=_safe_float(r.get("estimate_b")) if with_compare else None, - rel_error_b=_safe_float(r.get("rel_error_b")) if with_compare else None, - abs_rel_error_b=_safe_float(r.get("abs_rel_error_b")) if with_compare else None, - delta=_safe_float(r.get("delta")) if with_compare else None, - ) diff --git a/backend/routes/weights.py b/backend/routes/weights.py deleted file mode 100644 index ed6ae90..0000000 --- a/backend/routes/weights.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Weight distribution endpoints.""" - -import operator as op_module - -import numpy as np -from fastapi import APIRouter, Depends, HTTPException, Query - -from backend.state import get_state -from backend.models import ( - HistogramBin, - WeightDistribution, - WeightSlice, -) -from backend.state import AppState - -router = APIRouter() - -_OPS = { - "gt": op_module.gt, - "gte": op_module.ge, - "lt": op_module.lt, - "lte": op_module.le, - "eq": op_module.eq, - "ne": op_module.ne, -} - - -def _weight_stats(w: np.ndarray) -> dict: - """Compute standard weight distribution statistics.""" - if len(w) == 0: - return {"kish_effective_n": 0, "cv": 0, "design_effect": 1, "mean": 0, - "median": 0, "p5": 0, "p25": 0, "p75": 0, "p95": 0, "max": 0, - "top_1pct_weight_share": 0, "top_5pct_weight_share": 0} - - total = w.sum() - kish = float(total ** 2 / (w ** 2).sum()) if (w ** 2).sum() > 0 else 0 - mean = float(w.mean()) - std = float(w.std()) - cv = std / mean if mean > 0 else 0 - - sorted_w = np.sort(w)[::-1] - n = len(sorted_w) - top_1 = float(sorted_w[: max(1, n // 100)].sum() / total) if total > 0 else 0 - top_5 = float(sorted_w[: max(1, n // 20)].sum() / total) if total > 0 else 0 - - return { - "kish_effective_n": kish, - "cv": cv, - "design_effect": 1 + cv ** 2, - "mean": mean, - "median": float(np.median(w)), - "p5": float(np.percentile(w, 5)), - "p25": float(np.percentile(w, 25)), - "p75": float(np.percentile(w, 75)), - "p95": float(np.percentile(w, 95)), - "max": float(w.max()), - "top_1pct_weight_share": top_1 * 100, - "top_5pct_weight_share": top_5 * 100, - } - - -def _geo_mask(state: AppState, state_fips: int | None, cd_geoid: int | None) -> np.ndarray: - mask = np.ones(len(state.g_weights), dtype=bool) - if cd_geoid is not None: - mask = state.households_df["cd_geoid"].values == cd_geoid - elif state_fips is not None: - mask = state.households_df["state"].values == state_fips - return mask - - -@router.get("/distribution") -def weight_distribution( - slice_by: str = "none", - metric: str = "g_weight", - state_fips: int | None = Query(None, alias="state_fips"), - cd_geoid: int | None = Query(None, alias="cd_geoid"), - state: AppState = Depends(get_state), -) -> WeightDistribution: - metric_map = { - "g_weight": state.g_weights, - "final_weight": state.final_weights, - "initial_weight": state.initial_weights, - } - if metric not in metric_map: - raise HTTPException(status_code=400, detail=f"Unknown metric: {metric}") - - mask = _geo_mask(state, state_fips, cd_geoid) - w = metric_map[metric][mask] - overall = _weight_stats(w) - - slices = [] - if slice_by == "income_decile": - deciles = state.households_df["income_decile"].values - for d in range(10): - mask = deciles == d - s = _weight_stats(w[mask]) - slices.append(WeightSlice( - label=f"Decile {d + 1}", - n=int(mask.sum()), - kish_effective_n=s["kish_effective_n"], - mean=s["mean"], - median=s["median"], - )) - elif slice_by == "poverty_status": - for label, mask in [ - ("Poor", state.households_df["in_poverty"].values), - ("Non-poor", ~state.households_df["in_poverty"].values), - ]: - s = _weight_stats(w[mask]) - slices.append(WeightSlice( - label=label, - n=int(mask.sum()), - kish_effective_n=s["kish_effective_n"], - mean=s["mean"], - median=s["median"], - )) - elif slice_by == "state": - states = state.households_df["state"].values - for st in np.unique(states): - mask = states == st - s = _weight_stats(w[mask]) - slices.append(WeightSlice( - label=f"State {int(st):02d}", - n=int(mask.sum()), - kish_effective_n=s["kish_effective_n"], - mean=s["mean"], - median=s["median"], - )) - - return WeightDistribution(**overall, slices=slices) - - -@router.get("/histogram") -def weight_histogram( - metric: str = "g_weight", - bins: int = 50, - log_scale: bool = True, - filter_variable: str | None = None, - filter_operator: str = "gt", - filter_value: float = 0, - state: AppState = Depends(get_state), -) -> list[HistogramBin]: - metric_map = { - "g_weight": state.g_weights, - "final_weight": state.final_weights, - "initial_weight": state.initial_weights, - } - if metric not in metric_map: - raise HTTPException(status_code=400, detail=f"Unknown metric: {metric}") - - w = metric_map[metric].copy() - - if filter_variable: - if filter_operator not in _OPS: - raise HTTPException(status_code=400, detail=f"Invalid operator: {filter_operator}") - if not state.sim_service.variable_exists(filter_variable): - raise HTTPException(status_code=400, detail=f"Unknown variable: {filter_variable}") - vals = state.sim_service.calculate(filter_variable, map_to="household") - mask = _OPS[filter_operator](vals, filter_value) - w = w[mask] - - positive = w[w > 0] - if len(positive) == 0: - return [] - - if log_scale: - log_vals = np.log10(positive) - bin_edges = np.logspace(log_vals.min(), log_vals.max(), bins + 1) - else: - bin_edges = np.linspace(positive.min(), positive.max(), bins + 1) - - counts, edges = np.histogram(positive, bins=bin_edges) - return [ - HistogramBin( - bin_min=float(edges[i]), - bin_max=float(edges[i + 1]), - count=int(counts[i]), - ) - for i in range(len(counts)) - ] diff --git a/backend/scripts/extract_pipeline_dag.py b/backend/scripts/extract_pipeline_dag.py deleted file mode 100644 index 86f84e1..0000000 --- a/backend/scripts/extract_pipeline_dag.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Extract the pipeline DAG from policyengine_us_data via AST. - -Walks the package, finds every `@pipeline_node(...)` decorator, pulls its -fields (handling both forms: keyword args and PipelineNode(...) literal), -and emits a structured JSON consumed by the dashboard. - -Run from the repo root:: - - python3.12 backend/scripts/extract_pipeline_dag.py - -Output: backend/data/pipeline/nodes.json -""" - -from __future__ import annotations - -import ast -import importlib.util -import json -import logging -from collections import Counter -from pathlib import Path -from typing import Any - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") - - -def _locate_package_root() -> Path: - spec = importlib.util.find_spec("policyengine_us_data") - if spec is None or spec.origin is None: - raise RuntimeError("policyengine_us_data not installed") - return Path(spec.origin).parent - - -def _literal(node: ast.AST) -> Any: - """Best-effort literal eval, handling lists/tuples/dicts/strings/numbers.""" - try: - return ast.literal_eval(node) - except (ValueError, SyntaxError): - # Fall back to a readable representation - return ast.unparse(node) - - -def _extract_keywords(call: ast.Call) -> dict[str, Any]: - out: dict[str, Any] = {} - for kw in call.keywords: - if kw.arg is None: - continue - out[kw.arg] = _literal(kw.value) - return out - - -def _is_pipeline_node_decorator(deco: ast.AST) -> ast.Call | None: - """Return the inner Call node if `deco` is `@pipeline_node(...)`.""" - if not isinstance(deco, ast.Call): - return None - func = deco.func - name = getattr(func, "attr", None) if isinstance(func, ast.Attribute) else getattr(func, "id", None) - if name == "pipeline_node": - return deco - return None - - -def _node_from_decorator(deco_call: ast.Call) -> dict[str, Any]: - """Build a node dict from a @pipeline_node(...) decorator call. - - Handles both forms: - @pipeline_node(PipelineNode(id=..., label=..., ...)) - @pipeline_node(id=..., label=..., ...) - """ - # Form 1: positional PipelineNode(...) literal - if deco_call.args and isinstance(deco_call.args[0], ast.Call): - inner = deco_call.args[0] - if getattr(inner.func, "id", None) == "PipelineNode": - return _extract_keywords(inner) - # Form 2: kwargs directly on @pipeline_node - return _extract_keywords(deco_call) - - -def _decorator_target_name(func_or_class: ast.AST) -> str: - """The name of the function/class the decorator is attached to.""" - if isinstance(func_or_class, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - return func_or_class.name - return "" - - -# --- 5-stage canonical taxonomy --------------------------------------------- -# -# The team uses 5 stages declared in policyengine_us_data.stage_contracts.stages: -# STAGE_1_BUILD_DATASETS, STAGE_2_BUILD_CALIBRATION_PACKAGE, -# STAGE_3_FIT_WEIGHTS, STAGE_4_BUILD_OUTPUTS, STAGE_5_VALIDATE_AND_PROMOTE_RELEASE. -# -# Pipeline nodes don't carry stage_id directly, so we map via pathway + a -# heuristic that splits local_h5 into Stage 4 (build) vs Stage 5 (validate/ -# promote/release). - -STAGE_BY_PATHWAY = { - "data_build": "1_build_datasets", - "calibration_package": "2_build_calibration_package", - "weight_fit": "3_fit_weights", - "local_h5": "4_build_outputs", -} - -STAGE_5_KEYWORDS = ( - "validate_", "atomic_promote", "publish_", "_promote", - "release", "_sanity", "version_manifest", -) - - -def _stage_for_node(node: dict) -> str: - """Best-effort mapping from a node's pathway + id to one of the 5 - canonical stages. Local_h5 nodes that look like validate/promote work - are reclassified to Stage 5.""" - pathways = node.get("pathways") or [] - primary = pathways[0] if pathways else "" - base_stage = STAGE_BY_PATHWAY.get(primary, "") - - # Promote local_h5 validate/release nodes to Stage 5. - if base_stage == "4_build_outputs": - nid = node.get("id") or "" - src = node.get("source_file") or "" - if any(kw in nid for kw in STAGE_5_KEYWORDS) or any( - kw in src for kw in STAGE_5_KEYWORDS - ): - return "5_validate_and_promote_release" - return base_stage - - -def extract_nodes(package_root: Path) -> list[dict[str, Any]]: - nodes: list[dict[str, Any]] = [] - seen_ids: list[str] = [] - for py in sorted(package_root.rglob("*.py")): - if "__pycache__" in py.parts: - continue - try: - tree = ast.parse(py.read_text(encoding="utf-8")) - except SyntaxError as exc: - logger.warning("Skipping %s: %s", py, exc) - continue - for top in ast.walk(tree): - decos = getattr(top, "decorator_list", None) - if not decos: - continue - for deco in decos: - call = _is_pipeline_node_decorator(deco) - if call is None: - continue - node = _node_from_decorator(call) - if not node.get("id"): - # Skip the pure example decorators in pipeline_metadata.py - continue - rel_source = py.relative_to(package_root.parent).as_posix() - node.setdefault("source_file", rel_source) - node["target_symbol"] = _decorator_target_name(top) - node["decorator_line"] = deco.lineno - node["stage_id"] = _stage_for_node(node) - seen_ids.append(node["id"]) - nodes.append(node) - return nodes - - -def build_edges(nodes: list[dict]) -> list[dict]: - """Derive data-flow edges from artifacts_in/out. - - For every artifact A: node X produces A, node Y consumes A -> edge X→Y. - """ - by_id: dict[str, dict] = {n["id"]: n for n in nodes} - producers: dict[str, list[str]] = {} - consumers: dict[str, list[str]] = {} - - for n in nodes: - for art in (n.get("artifacts_out") or []): - producers.setdefault(art, []).append(n["id"]) - for art in (n.get("artifacts_in") or []): - consumers.setdefault(art, []).append(n["id"]) - - edges: list[dict] = [] - seen: set[tuple[str, str, str]] = set() - for art, prods in producers.items(): - for cons_id in consumers.get(art, []): - for prod_id in prods: - if prod_id == cons_id: - continue - key = (prod_id, cons_id, art) - if key in seen: - continue - seen.add(key) - edges.append({ - "from": prod_id, - "to": cons_id, - "artifact": art, - "kind": "data_flow", - }) - - # Artifacts that have no producer (purely external) - unproduced = sorted(set(consumers.keys()) - set(producers.keys())) - return edges, unproduced - - -def main(): - pkg_root = _locate_package_root() - logger.info("Scanning %s", pkg_root) - nodes = extract_nodes(pkg_root) - edges, unproduced = build_edges(nodes) - - # Stats - by_type = Counter(n.get("node_type", "?") for n in nodes) - by_status = Counter(n.get("status", "?") for n in nodes) - by_pathway = Counter() - for n in nodes: - for p in (n.get("pathways") or []): - by_pathway[p] += 1 - by_stage = Counter(n.get("stage_id") or "(unknown)" for n in nodes) - - logger.info("Found %d nodes", len(nodes)) - logger.info(" by type: %s", dict(by_type)) - logger.info(" by status: %s", dict(by_status)) - logger.info(" by pathway: %s", dict(by_pathway)) - logger.info(" by stage: %s", dict(by_stage)) - logger.info("Built %d edges (data-flow). Unproduced artifacts: %d", - len(edges), len(unproduced)) - - out_dir = Path("backend/data/pipeline") - out_dir.mkdir(parents=True, exist_ok=True) - payload = { - "nodes": nodes, - "edges": edges, - "unproduced_artifacts": unproduced, - "stats": { - "node_count": len(nodes), - "edge_count": len(edges), - "by_type": dict(by_type), - "by_status": dict(by_status), - "by_pathway": dict(by_pathway), - "by_stage": dict(by_stage), - }, - } - out_path = out_dir / "nodes.json" - out_path.write_text(json.dumps(payload, indent=2, sort_keys=True)) - logger.info("Wrote %s", out_path) - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/target_index/__init__.py b/backend/scripts/target_index/__init__.py deleted file mode 100644 index f9bed10..0000000 --- a/backend/scripts/target_index/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Audit of all 5 target storage tiers in policyengine_us_data. - -Targets live in five distinct places, and policy_data.db (what the dashboard -reads today) is only the compiled output of a subset: - -1. storage/calibration_targets/*.csv hand-edited canonical source values -2. storage/calibration_targets/*.py programmatic generators (aca_ptc_targets.py) -3. utils/*.py hard-coded constants (cms_medicare, takeup, ...) -4. calibration/target_config.yaml opt-in rules (no values, just inclusion logic) -5. policy_data.db the compiled DB shipped on HF - -This module produces backend/data/target_index.json — the cross-tier union with -provenance per target — and an audit report comparing each tier against tier 5. -""" diff --git a/backend/scripts/target_index/audit.py b/backend/scripts/target_index/audit.py deleted file mode 100644 index 26bbd79..0000000 --- a/backend/scripts/target_index/audit.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Audit orchestrator: run every registered parser, match against the DB, -emit backend/data/target_index.json + a human-readable summary on stdout. - -Usage: - python -m backend.scripts.target_index.audit \\ - --calibration-targets ".artifacts/.../storage/calibration_targets" \\ - --db ".artifacts/.../policy_data.db" \\ - --out backend/data/target_index.json - -Both --calibration-targets and --db default to the installed-package locations. -""" - -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path - -from backend.scripts.target_index.matcher import ( - cross_tier_audit, - load_db_targets, -) -from backend.scripts.target_index.parsers import PARSER_REGISTRY -from backend.scripts.target_index.schema import TargetRecord - -logging.basicConfig(level=logging.INFO, format="%(levelname)-5s %(message)s") -logger = logging.getLogger("audit") - - -def _default_calibration_targets() -> Path: - """The installed policyengine_us_data ships these CSVs under storage/.""" - try: - import policyengine_us_data - root = Path(policyengine_us_data.__file__).parent - return root / "storage" / "calibration_targets" - except ImportError: - return Path() - - -def _default_db_path() -> Path: - """Use the cached HF artifact if available.""" - candidates = [ - Path(".artifacts/PolicyEngine__policyengine-us-data-pipeline/test/policy_data.db"), - Path(".artifacts/PolicyEngine__policyengine-us-data/staging/usdata-gha25719239158-a1-889ab438/policy_data.db"), - ] - for p in candidates: - if p.exists(): - return p - return Path() - - -def run_audit(csv_dir: Path, db_path: Path, out_path: Path) -> dict: - if not csv_dir.exists(): - logger.error("calibration_targets dir not found: %s", csv_dir) - sys.exit(1) - if not db_path.exists(): - logger.error("policy_data.db not found: %s", db_path) - sys.exit(1) - - # 1. Discover every CSV in the directory - all_csvs = sorted(p.name for p in csv_dir.glob("*.csv")) - logger.info("Found %d CSV files in %s", len(all_csvs), csv_dir) - - # 2. Run each registered parser - tier_records: dict[str, list[TargetRecord]] = {} - parsed_total = 0 - for fname in all_csvs: - parser = PARSER_REGISTRY.get(fname) - if parser is None: - logger.warning(" ⚠ no parser for %s (skipping)", fname) - continue - records = parser(csv_dir / fname) - tier_records[f"csv/{fname}"] = records - parsed_total += len(records) - logger.info(" ✓ %s → %d records", fname, len(records)) - - # 2b. Tier 3 — Python constants - from backend.scripts.target_index import python_constants - py_records = python_constants.collect() - if py_records: - tier_records["python/cms_medicare+takeup"] = py_records - logger.info(" ✓ python constants → %d records", len(py_records)) - - # 3. Load the DB tier - logger.info("Loading policy_data.db: %s", db_path) - db_records = load_db_targets(db_path) - logger.info(" → %d DB targets", len(db_records)) - - # 4. Match - audit = cross_tier_audit(tier_records, db_records) - - # 5. Persist outputs: - # - → audit summary only (small, committed) - # - .union.jsonl → full per-record dump (large, gitignored) - out_path.parent.mkdir(parents=True, exist_ok=True) - summary = { - "db_total": audit["db_total"], - "tiers": audit["tiers"], - "parsers_covered": sorted(PARSER_REGISTRY.keys()), - "parsers_missing": sorted(set(all_csvs) - set(PARSER_REGISTRY.keys())), - } - out_path.write_text(json.dumps(summary, indent=2, default=str)) - logger.info("Wrote audit summary → %s", out_path) - - # Full per-record dump as JSONL (one record per line) — large, gitignored. - union_path = out_path.with_suffix(out_path.suffix + ".union.jsonl") - with union_path.open("w") as f: - for rs in tier_records.values(): - for r in rs: - f.write(json.dumps(r.to_dict(), default=str) + "\n") - for r in db_records: - f.write(json.dumps(r.to_dict(), default=str) + "\n") - logger.info("Wrote full union → %s (gitignored)", union_path) - - return audit - - -def _print_summary(audit: dict) -> None: - print() - print("=" * 60) - print("TARGET-INDEX AUDIT") - print("=" * 60) - print(f"DB total active targets: {audit['db_total']:,}") - print() - print(f"{'Tier':40} {'records':>8} {'matched':>8} {'%':>6}") - print("-" * 70) - for t in audit["tiers"]: - rate = (t["match_rate"] or 0) * 100 - print(f"{t['tier']:40} {t['total_records']:>8,} " - f"{t['matched_to_db']:>8,} {rate:>5.1f}%") - - print() - for t in audit["tiers"]: - if t["unmatched"] == 0: continue - print(f"--- {t['tier']}: {t['unmatched']} unmatched (showing up to 5) ---") - for ex in t["unmatched_examples"][:5]: - print(f" variable={ex['variable']!r} period={ex.get('period')} " - f"geo={ex.get('geo_level')}/{ex.get('geographic_id')} " - f"cons={ex.get('constraints')}") - print() - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--calibration-targets", type=Path, - default=_default_calibration_targets()) - ap.add_argument("--db", type=Path, default=_default_db_path()) - ap.add_argument("--out", type=Path, - default=Path("backend/data/target_index.json")) - args = ap.parse_args() - audit = run_audit(args.calibration_targets, args.db, args.out) - _print_summary(audit) - - -if __name__ == "__main__": - main() diff --git a/backend/scripts/target_index/matcher.py b/backend/scripts/target_index/matcher.py deleted file mode 100644 index bca484b..0000000 --- a/backend/scripts/target_index/matcher.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Match TargetRecords from tiers 1-4 against policy_data.db (tier 5).""" - -from __future__ import annotations - -import logging -import sqlite3 -from collections import defaultdict -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -logger = logging.getLogger(__name__) - - -def load_db_targets(db_path: Path) -> list[TargetRecord]: - """Read every active target from policy_data.db into TargetRecord form, - so the matcher can compare apples-to-apples by signature.""" - conn = sqlite3.connect(str(db_path)) - conn.row_factory = sqlite3.Row - - # Pull all targets + their stratum constraints in two passes. - targets = list(conn.execute( - "SELECT target_id, variable, period, stratum_id, value, active, " - "source, notes FROM targets WHERE active = 1" - )) - constraints = defaultdict(list) - for c in conn.execute( - "SELECT stratum_id, constraint_variable, operation, value " - "FROM stratum_constraints" - ): - constraints[c["stratum_id"]].append(( - c["constraint_variable"], c["operation"], c["value"], - )) - conn.close() - - geo_vars = {"state_fips", "congressional_district_geoid", "ucgid_str"} - out: list[TargetRecord] = [] - for t in targets: - cons_for_stratum = constraints.get(t["stratum_id"], []) - geo_level: str | None = None - gid: str | None = None - non_geo: list[tuple[str, str, str]] = [] - for cvar, op, cval in cons_for_stratum: - if cvar in geo_vars and op == "==": - if cvar == "state_fips": - geo_level = "state" - elif cvar == "congressional_district_geoid": - geo_level = "district" - else: - geo_level = cvar - gid = str(cval) - else: - non_geo.append((cvar, op, str(cval))) - - if geo_level is None: - geo_level = "national" - - out.append(TargetRecord( - variable=t["variable"], - geo_level=geo_level, - geographic_id=gid, - period=t["period"], - constraints=tuple(sorted(non_geo)), - value=t["value"], - is_count=False, # the DB doesn't flag count vs amount - storage_tier="db", - source_path="policy_data.db", - source_row=f"target_id={t['target_id']}/stratum_id={t['stratum_id']}", - notes=(t["notes"] or ""), - )) - return out - - -def index_by_signature(records: list[TargetRecord]) -> dict[tuple, list[TargetRecord]]: - by_sig: dict[tuple, list[TargetRecord]] = defaultdict(list) - for r in records: - by_sig[r.signature()].append(r) - return by_sig - - -def cross_tier_audit( - tier_records: dict[str, list[TargetRecord]], - db_records: list[TargetRecord], -) -> dict: - """For each non-DB tier, report: how many of its records signature-match - a DB target. Returns a structured audit. - - tier_records: {tier_label: [records]} from tiers 1-4. - db_records: tier 5 (loaded via load_db_targets). - """ - db_by_sig = index_by_signature(db_records) - audit = { - "db_total": len(db_records), - "tiers": [], - } - for tier_label, records in tier_records.items(): - # Aggregate the rare case of duplicate signatures within a source - per_sig: dict[tuple, list[TargetRecord]] = defaultdict(list) - for r in records: - per_sig[r.signature()].append(r) - matched, unmatched = [], [] - for sig, rs in per_sig.items(): - if sig in db_by_sig: - matched.extend(rs) - else: - unmatched.extend(rs) - audit["tiers"].append({ - "tier": tier_label, - "total_records": len(records), - "unique_signatures": len(per_sig), - "matched_to_db": len(matched), - "unmatched": len(unmatched), - "match_rate": (len(matched) / len(records)) if records else None, - "unmatched_examples": [r.to_dict() for r in unmatched[:5]], - }) - return audit diff --git a/backend/scripts/target_index/parsers/__init__.py b/backend/scripts/target_index/parsers/__init__.py deleted file mode 100644 index fe6416b..0000000 --- a/backend/scripts/target_index/parsers/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Per-CSV parsers for storage/calibration_targets/*.csv. - -Each module exposes `parse(csv_path: Path) -> list[TargetRecord]`. - -Add a new parser by: -1. Drop a module in this directory. -2. Register it in PARSER_REGISTRY below, keyed by CSV filename. - -The audit orchestrator (audit.py) iterates the registry and invokes each -parser, then matches the resulting records against policy_data.db. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Callable - -from backend.scripts.target_index.schema import TargetRecord - -from . import ( - aca_marketplace_state_metal_selection_2024, - aca_ptc_state, - aca_spending_and_enrollment_2024, - aca_spending_and_enrollment_2025, - aca_spending_and_enrollment_2026, - acs_housing_costs_2024, - age_state, - agi_state, - eitc_by_agi_and_children, - eitc_claim_controls, - eitc_state, - healthcare_spending, - medicaid_enrollment_2024, - medicaid_enrollment_2025, - medicaid_enrollment_2026, - np2023_d5_mid, - population_by_state, - real_estate_taxes_by_state_acs, - snap_state, - soi_targets, - spm_threshold_agi, -) - -PARSER_REGISTRY: dict[str, Callable[[Path], list[TargetRecord]]] = { - "aca_marketplace_state_metal_selection_2024.csv": aca_marketplace_state_metal_selection_2024.parse, - "aca_ptc_state.csv": aca_ptc_state.parse, - "aca_spending_and_enrollment_2024.csv": aca_spending_and_enrollment_2024.parse, - "aca_spending_and_enrollment_2025.csv": aca_spending_and_enrollment_2025.parse, - "aca_spending_and_enrollment_2026.csv": aca_spending_and_enrollment_2026.parse, - "acs_housing_costs_2024.csv": acs_housing_costs_2024.parse, - "age_state.csv": age_state.parse, - "agi_state.csv": agi_state.parse, - "eitc_by_agi_and_children.csv": eitc_by_agi_and_children.parse, - "eitc_claim_controls.csv": eitc_claim_controls.parse, - "eitc_state.csv": eitc_state.parse, - "healthcare_spending.csv": healthcare_spending.parse, - "medicaid_enrollment_2024.csv": medicaid_enrollment_2024.parse, - "medicaid_enrollment_2025.csv": medicaid_enrollment_2025.parse, - "medicaid_enrollment_2026.csv": medicaid_enrollment_2026.parse, - "np2023_d5_mid.csv": np2023_d5_mid.parse, - "population_by_state.csv": population_by_state.parse, - "real_estate_taxes_by_state_acs.csv": real_estate_taxes_by_state_acs.parse, - "snap_state.csv": snap_state.parse, - "soi_targets.csv": soi_targets.parse, - "spm_threshold_agi.csv": spm_threshold_agi.parse, -} diff --git a/backend/scripts/target_index/parsers/aca_marketplace_state_metal_selection_2024.py b/backend/scripts/target_index/parsers/aca_marketplace_state_metal_selection_2024.py deleted file mode 100644 index 7e0830a..0000000 --- a/backend/scripts/target_index/parsers/aca_marketplace_state_metal_selection_2024.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Parser for storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv. - -CMS 2024 OEP State-Metal-Status PUF. 307 rows with columns: - year, source, state_code, platform, metal_level, enrollment_status, - consumers, avg_selected_premium, avg_selected_net_premium, - selected_lte10_share, selected_lte10_consumers, aptc_consumers, - aptc_share, avg_aptc - -The file mixes two breakdown styles per (state, platform): - - metal_level != "All", enrollment_status == "All" → split by metal tier - - metal_level == "All", enrollment_status != "All" → split by enrollment status - -Schema decision (documented for the audit reviewer): -We emit targets ONLY for the (state, metal_level) breakdown rows where -metal_level in {B, S, G} (Bronze/Silver/Gold) and enrollment_status == "All". -For each surviving row we sum across the two platforms (HC.gov + SBM) and -emit TWO targets per (state, metal_level): - - consumers → tax_unit_count with aca_ptc >= 0 and metal_level constraint - - aptc_consumers → tax_unit_count with aca_ptc > 0 and metal_level constraint - -The metal_level constraint maps to PolicyEngine variable `aca_metal_level` -with values "bronze" / "silver" / "gold" (best-guess canonical names — the -PE codebase exposes a categorical metal-tier enum; if the audit reports -these as unmatched, the constraint variable name is the most likely culprit -and can be retuned without touching the row count). - -Platform breakdown (HC.gov vs SBM) is collapsed because the calibration -target is the total state population per metal tier, not a marketplace-type -split. Period=2024. -""" - -from __future__ import annotations - -import csv -from collections import defaultdict -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/aca_marketplace_state_metal_selection_2024.csv" -PERIOD = 2024 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - -# Single-letter metal codes in the PUF → PolicyEngine metal-level values. -METAL_CODE_TO_NAME = { - "B": "bronze", - "S": "silver", - "G": "gold", - # "C" (catastrophic) / "P" (platinum) do not appear in this file. -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - # Aggregate across platforms keyed by (state_fips, metal_code). - # Each accumulator stores (consumers_sum, aptc_consumers_sum, first_row_key). - aggregates: dict[tuple[str, str], dict] = defaultdict( - lambda: {"consumers": 0.0, "aptc_consumers": 0.0, "first_row": None} - ) - - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - metal = row["metal_level"].strip() - status = row["enrollment_status"].strip() - if metal == "All" or status != "All": - # Skip the enrollment-status breakdown rows. - continue - if metal not in METAL_CODE_TO_NAME: - continue - - state = row["state_code"].strip().upper() - gid = STATE_CODE_TO_FIPS.get(state) - if gid is None: - continue - - try: - consumers = float(row["consumers"]) - aptc_consumers = float(row["aptc_consumers"]) - except (TypeError, ValueError): - continue - - key = (gid, metal) - agg = aggregates[key] - agg["consumers"] += consumers - agg["aptc_consumers"] += aptc_consumers - if agg["first_row"] is None: - agg["first_row"] = f"row-{i + 2}" - - out: list[TargetRecord] = [] - for (gid, metal_code), agg in aggregates.items(): - metal_name = METAL_CODE_TO_NAME[metal_code] - row_key = agg["first_row"] or "row-?" - - # Total marketplace consumers in this metal tier (includes non-PTC). - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=( - ("aca_metal_level", "==", metal_name), - ("tax_unit_is_filer", "==", "1"), - ), - value=agg["consumers"], - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"CMS 2024 OEP PUF — marketplace consumers, metal={metal_name}", - )) - # PTC-receiving consumers in this metal tier. - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=( - ("aca_metal_level", "==", metal_name), - ("aca_ptc", ">", "0"), - ("tax_unit_is_filer", "==", "1"), - ), - value=agg["aptc_consumers"], - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"CMS 2024 OEP PUF — APTC consumers, metal={metal_name}", - )) - return out diff --git a/backend/scripts/target_index/parsers/aca_ptc_state.py b/backend/scripts/target_index/parsers/aca_ptc_state.py deleted file mode 100644 index ffb458d..0000000 --- a/backend/scripts/target_index/parsers/aca_ptc_state.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Parser for storage/calibration_targets/aca_ptc_state.csv. - -Schema (after the leading `#`-comment line): GEO_ID, Returns, TotalPTCAmount. -Source: IRS SOI Historical Table 2 (TY2022), columns N85770 / A85770. - -Each row produces TWO target records: -- Returns: count of tax units with aca_ptc > 0 (variable=tax_unit_count) -- TotalPTCAmount: dollar amount of aca_ptc - -Both are filer-gated (tax_unit_is_filer == 1). State-level via the GEO_ID -`0400000US` convention (leading zeros stripped to mirror DB ints). - -Period defaults to 2022 (the SOI tax year for this file). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/aca_ptc_state.csv" -PERIOD = 2022 - - -def _geo_id_to_geo(geo_id: str) -> tuple[str, str | None]: - """0400000US01 → ('state', '1'). 0100000US → ('national', None).""" - geo_id = geo_id.strip() - if geo_id.startswith("0400000US"): - fips_raw = geo_id[len("0400000US"):] - return "state", str(int(fips_raw)) if fips_raw.isdigit() else fips_raw - if geo_id.startswith("0100000US"): - return "national", None - return "national", None - - -def _uncommented(lines): - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - yield line - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(_uncommented(f)) - for i, row in enumerate(reader): - geo_level, gid = _geo_id_to_geo(row["GEO_ID"]) - returns = float(row["Returns"]) - amount = float(row["TotalPTCAmount"]) - row_key = f"row-{i + 2}" # +2 for header + comment + 1-indexed-ish - - # Count of filers receiving PTC - out.append(TargetRecord( - variable="tax_unit_count", - geo_level=geo_level, - geographic_id=gid, - period=PERIOD, - constraints=( - ("aca_ptc", ">", "0"), - ("tax_unit_is_filer", "==", "1"), - ), - value=returns, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="IRS SOI Historical Table 2 — PTC returns (N85770)", - )) - # Dollar amount of PTC - out.append(TargetRecord( - variable="aca_ptc", - geo_level=geo_level, - geographic_id=gid, - period=PERIOD, - constraints=(("tax_unit_is_filer", "==", "1"),), - value=amount, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="IRS SOI Historical Table 2 — PTC amount (A85770, $)", - )) - return out diff --git a/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2024.py b/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2024.py deleted file mode 100644 index 1427261..0000000 --- a/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2024.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Parser for storage/calibration_targets/aca_spending_and_enrollment_2024.csv. - -Schema: state, enrollment, spending. State codes are USPS two-letter (e.g. AK). -Each row produces TWO target records at state level: -- enrollment: count of tax units with aca_ptc > 0 (variable=tax_unit_count) -- spending: dollar amount of aca_ptc - -Both filer-gated. Period=2024. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/aca_spending_and_enrollment_2024.csv" -PERIOD = 2024 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip().upper() - gid = STATE_CODE_TO_FIPS.get(state) - if gid is None: - continue - enrollment = float(row["enrollment"]) - spending = float(row["spending"]) - row_key = f"row-{i + 2}" - - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=( - ("aca_ptc", ">", "0"), - ("tax_unit_is_filer", "==", "1"), - ), - value=enrollment, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="ACA marketplace enrollment (PTC recipients), 2024", - )) - out.append(TargetRecord( - variable="aca_ptc", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=(("tax_unit_is_filer", "==", "1"),), - value=spending, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="ACA PTC spending ($), 2024", - )) - return out diff --git a/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2025.py b/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2025.py deleted file mode 100644 index 14caeda..0000000 --- a/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2025.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Parser for storage/calibration_targets/aca_spending_and_enrollment_2025.csv. - -Identical schema/semantics to the 2024 file, with period=2025. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/aca_spending_and_enrollment_2025.csv" -PERIOD = 2025 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip().upper() - gid = STATE_CODE_TO_FIPS.get(state) - if gid is None: - continue - enrollment = float(row["enrollment"]) - spending = float(row["spending"]) - row_key = f"row-{i + 2}" - - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=( - ("aca_ptc", ">", "0"), - ("tax_unit_is_filer", "==", "1"), - ), - value=enrollment, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="ACA marketplace enrollment (PTC recipients), 2025", - )) - out.append(TargetRecord( - variable="aca_ptc", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=(("tax_unit_is_filer", "==", "1"),), - value=spending, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="ACA PTC spending ($), 2025", - )) - return out diff --git a/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2026.py b/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2026.py deleted file mode 100644 index 7776bed..0000000 --- a/backend/scripts/target_index/parsers/aca_spending_and_enrollment_2026.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Parser for storage/calibration_targets/aca_spending_and_enrollment_2026.csv. - -Identical schema/semantics to the 2024 file, with period=2026. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/aca_spending_and_enrollment_2026.csv" -PERIOD = 2026 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip().upper() - gid = STATE_CODE_TO_FIPS.get(state) - if gid is None: - continue - enrollment = float(row["enrollment"]) - spending = float(row["spending"]) - row_key = f"row-{i + 2}" - - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=( - ("aca_ptc", ">", "0"), - ("tax_unit_is_filer", "==", "1"), - ), - value=enrollment, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="ACA marketplace enrollment (PTC recipients), 2026", - )) - out.append(TargetRecord( - variable="aca_ptc", - geo_level="state", - geographic_id=gid, - period=PERIOD, - constraints=(("tax_unit_is_filer", "==", "1"),), - value=spending, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="ACA PTC spending ($), 2026", - )) - return out diff --git a/backend/scripts/target_index/parsers/acs_housing_costs_2024.py b/backend/scripts/target_index/parsers/acs_housing_costs_2024.py deleted file mode 100644 index 033275d..0000000 --- a/backend/scripts/target_index/parsers/acs_housing_costs_2024.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Parser for storage/calibration_targets/acs_housing_costs_2024.csv. - -Schema: state_code, state_fips, annual_contract_rent, real_estate_taxes. -51 rows. State-level dollar amounts (already in dollars, no scaling). - -Each row → TWO targets: - 1. rent ($ annual_contract_rent), state-level. - 2. real_estate_taxes ($ real_estate_taxes), state-level. - -period=2024. is_count=False for both. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/acs_housing_costs_2024.csv" -PERIOD = 2024 - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - # state_fips column may be zero-padded ("02") — normalise to bare int-string. - fips_raw = row["state_fips"].strip() - try: - fips = str(int(fips_raw)) - except ValueError: - continue - rent = float(row["annual_contract_rent"]) - ret = float(row["real_estate_taxes"]) - row_key = f"row-{i + 2}" - - out.append(TargetRecord( - variable="rent", - geo_level="state", - geographic_id=fips, - period=PERIOD, - value=rent, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"{row_key}/rent", - notes="ACS 2024 — annual contract rent (state aggregate, $)", - )) - out.append(TargetRecord( - variable="real_estate_taxes", - geo_level="state", - geographic_id=fips, - period=PERIOD, - value=ret, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"{row_key}/real_estate_taxes", - notes="ACS 2024 — real estate taxes (state aggregate, $)", - )) - return out diff --git a/backend/scripts/target_index/parsers/age_state.py b/backend/scripts/target_index/parsers/age_state.py deleted file mode 100644 index 72374c8..0000000 --- a/backend/scripts/target_index/parsers/age_state.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Parser for storage/calibration_targets/age_state.csv. - -Schema (wide): GEO_ID, GEO_NAME, 0-4, 5-9, 10-14, ..., 80-84, 85+ — 51 rows -(50 states + DC). Each cell is a population count for that state × age band. - -Each cell → ONE target: - variable = person_count - geo_level = state - geographic_id = state_fips (extracted from GEO_ID like 0400000US01 → "1") - constraints = age >= lo AND age < hi (open-ended for "85+": age >= 85) - is_count = True - period = 2024 (the file is undated; mirror the DB period family). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/age_state.csv" -PERIOD = 2024 - - -def _parse_band(band: str) -> tuple[int, int | None]: - """'0-4' → (0, 5); '85+' → (85, None).""" - band = band.strip() - if band.endswith("+"): - return int(band[:-1]), None - lo_s, hi_s = band.split("-") - return int(lo_s), int(hi_s) + 1 # "0-4" inclusive == age < 5 - - -def _geo_id_to_fips(geo_id: str) -> str: - """0400000US01 → '1' (strip leading zero, consistent with schema._norm_geo_id).""" - geo_id = geo_id.strip() - if geo_id.startswith("0400000US"): - return str(int(geo_id[len("0400000US"):])) - return geo_id - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.reader(f) - header = next(reader) - band_cols = header[2:] # skip GEO_ID, GEO_NAME - bands = [(col, _parse_band(col)) for col in band_cols] - - for i, row in enumerate(reader): - if not row: - continue - fips = _geo_id_to_fips(row[0]) - for col_idx, (band_label, (lo, hi)) in enumerate(bands): - raw = row[2 + col_idx].strip() - if not raw: - continue - value = float(raw) - cons: list[tuple[str, str, str]] = [("age", ">=", str(lo))] - if hi is not None: - cons.append(("age", "<", str(hi))) - out.append(TargetRecord( - variable="person_count", - geo_level="state", - geographic_id=fips, - period=PERIOD, - constraints=tuple(cons), - value=value, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"row-{i + 2}/{band_label}", - notes=f"Census ACS age × state — band {band_label}", - )) - return out diff --git a/backend/scripts/target_index/parsers/agi_state.py b/backend/scripts/target_index/parsers/agi_state.py deleted file mode 100644 index 449d7b6..0000000 --- a/backend/scripts/target_index/parsers/agi_state.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Parser for storage/calibration_targets/agi_state.csv. - -Schema: `GEO_ID, GEO_NAME, AGI_LOWER_BOUND, AGI_UPPER_BOUND, VALUE, IS_COUNT, -VARIABLE`. ~1,021 rows. State-level, AGI-banded. - -The VARIABLE column carries one of two PE-style values: -- `adjusted_gross_income/amount` — dollar AGI sum within band (IS_COUNT=0). -- `adjusted_gross_income/count` — number of filer tax units with AGI in band - (IS_COUNT=1). We map this to PE `tax_unit_count` with an extra - `adjusted_gross_income > 0` constraint so the count is "filers with AGI in - the band" — matching the DB-side convention used for similar SOI strata. - -All rows are filer-gated (`tax_unit_is_filer == 1`). - -The CSV is undated; we default to period=2022 (IRS-SOI-aligned). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/agi_state.csv" -PERIOD = 2022 - - -def _parse_bound(s: str) -> float: - s = s.strip() - if s in ("-inf", "-Infinity"): - return float("-inf") - if s in ("inf", "Infinity"): - return float("inf") - return float(s) - - -def _geo_id_to_state_fips(geo_id: str) -> str | None: - geo_id = geo_id.strip() - if geo_id.startswith("0400000US"): - return geo_id[len("0400000US"):] - return None - - -def _truthy(val) -> bool: - if isinstance(val, bool): - return val - if val is None: - return False - s = str(val).strip().lower() - return s in ("true", "1", "yes") - - -def _band_constraints(lo: float, hi: float) -> list[tuple[str, str, str]]: - cons: list[tuple[str, str, str]] = [] - if lo != float("-inf"): - cons.append(("adjusted_gross_income", ">=", str(lo))) - if hi != float("inf"): - cons.append(("adjusted_gross_income", "<", str(hi))) - return cons - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state_fips = _geo_id_to_state_fips(row["GEO_ID"]) - if state_fips is None: - continue - try: - lo = _parse_bound(row["AGI_LOWER_BOUND"]) - hi = _parse_bound(row["AGI_UPPER_BOUND"]) - value = float(row["VALUE"]) - except (TypeError, ValueError): - continue - - is_count = _truthy(row.get("IS_COUNT")) - raw_var = row["VARIABLE"].strip() - - # Strip the "/count" or "/amount" suffix and route accordingly. - cons: list[tuple[str, str, str]] = [("tax_unit_is_filer", "==", "1")] - if raw_var.endswith("/count"): - # Count of filer tax units within the AGI band. We re-map the - # variable to `tax_unit_count`; the AGI-band constraints below - # carry the "with AGI in [lo, hi)" semantics. - variable = "tax_unit_count" - elif raw_var.endswith("/amount"): - variable = raw_var[: -len("/amount")] - else: - variable = raw_var - cons.extend(_band_constraints(lo, hi)) - - row_key = f"{row['GEO_ID']}/{raw_var}/{row['AGI_LOWER_BOUND']}-{row['AGI_UPPER_BOUND']}#{i}" - - out.append(TargetRecord( - variable=variable, - geo_level="state", - geographic_id=state_fips, - period=PERIOD, - constraints=tuple(cons), - value=value, - is_count=is_count, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"IRS-SOI state AGI band ({raw_var})", - )) - return out diff --git a/backend/scripts/target_index/parsers/eitc_by_agi_and_children.py b/backend/scripts/target_index/parsers/eitc_by_agi_and_children.py deleted file mode 100644 index 9477910..0000000 --- a/backend/scripts/target_index/parsers/eitc_by_agi_and_children.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Parser for storage/calibration_targets/eitc_by_agi_and_children.csv. - -IRS SOI Publication 1304 Table 2.5, TY2022. EITC broken down by AGI band × -qualifying-child count. First line is a `#`-prefixed citation; header is -`count_children,agi_lower,agi_upper,returns,amount`. - -`count_children=3` means "three or more" — so we emit `eitc_child_count >= 3` -in that case, otherwise an exact `== ` match. - -Each row yields TWO national-level targets: -- `tax_unit_count` (count, constrained by `eitc > 0`). -- `eitc` dollar amount. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/eitc_by_agi_and_children.csv" -PERIOD = 2022 - - -def _parse_bound(s: str) -> float: - s = s.strip() - if s in ("-inf", "-Infinity"): - return float("-inf") - if s in ("inf", "Infinity"): - return float("inf") - return float(s) - - -def _uncommented(lines): - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - yield line - - -def _base_constraints(count_children: int, lo: float, hi: float) -> list[tuple[str, str, str]]: - cons: list[tuple[str, str, str]] = [ - ("tax_unit_is_filer", "==", "1"), - ("eitc", ">", "0"), - ] - # count_children=3 in the source means "three or more". - if count_children >= 3: - cons.append(("eitc_child_count", ">=", "3")) - else: - cons.append(("eitc_child_count", "==", str(count_children))) - if lo != float("-inf"): - cons.append(("adjusted_gross_income", ">=", str(lo))) - if hi != float("inf"): - cons.append(("adjusted_gross_income", "<", str(hi))) - return cons - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(_uncommented(f)) - for i, row in enumerate(reader): - try: - count_children = int(row["count_children"]) - lo = _parse_bound(row["agi_lower"]) - hi = _parse_bound(row["agi_upper"]) - returns = float(row["returns"]) - amount = float(row["amount"]) - except (TypeError, ValueError): - continue - - cons = tuple(_base_constraints(count_children, lo, hi)) - row_key = f"c{count_children}/{row['agi_lower']}-{row['agi_upper']}#{i}" - - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="national", - geographic_id=None, - period=PERIOD, - constraints=cons, - value=returns, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="IRS SOI Pub 1304 Table 2.5 (TY2022) — EITC returns by AGI×kids", - )) - out.append(TargetRecord( - variable="eitc", - geo_level="national", - geographic_id=None, - period=PERIOD, - constraints=cons, - value=amount, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="IRS SOI Pub 1304 Table 2.5 (TY2022) — EITC amount by AGI×kids", - )) - return out diff --git a/backend/scripts/target_index/parsers/eitc_claim_controls.py b/backend/scripts/target_index/parsers/eitc_claim_controls.py deleted file mode 100644 index 8ae5775..0000000 --- a/backend/scripts/target_index/parsers/eitc_claim_controls.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Parser for storage/calibration_targets/eitc_claim_controls.csv. - -IRS TY2024 EITC claim controls. First line is a `#`-prefixed citation; the -header is `year,GEO_ID,Returns,Amount`. Includes a national-level row -(GEO_ID `0100000US`) alongside the 51 state rows. - -Each row yields TWO target records: -- `tax_unit_count` (count) constrained by `eitc > 0` and filer gate. -- `eitc` dollar amount, filer-gated. - -Period comes from the `year` column (typically 2024). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/eitc_claim_controls.csv" - - -def _geo_id_to_geo(geo_id: str) -> tuple[str, str | None]: - geo_id = geo_id.strip() - if geo_id.startswith("0400000US"): - return "state", geo_id[len("0400000US"):] - if geo_id.startswith("0100000US"): - return "national", None - return "national", None - - -def _uncommented(lines): - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - yield line - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(_uncommented(f)) - for i, row in enumerate(reader): - geo_level, gid = _geo_id_to_geo(row["GEO_ID"]) - try: - year = int(row["year"]) - returns = float(row["Returns"]) - amount = float(row["Amount"]) - except (TypeError, ValueError): - continue - row_key = f"{row['GEO_ID']}#{year}#{i}" - - out.append(TargetRecord( - variable="tax_unit_count", - geo_level=geo_level, - geographic_id=gid, - period=year, - constraints=( - ("tax_unit_is_filer", "==", "1"), - ("eitc", ">", "0"), - ), - value=returns, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"IRS EITC claim controls TY{year} — returns", - )) - out.append(TargetRecord( - variable="eitc", - geo_level=geo_level, - geographic_id=gid, - period=year, - constraints=( - ("tax_unit_is_filer", "==", "1"), - ), - value=amount, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"IRS EITC claim controls TY{year} — amount", - )) - return out diff --git a/backend/scripts/target_index/parsers/eitc_state.py b/backend/scripts/target_index/parsers/eitc_state.py deleted file mode 100644 index 1dbbd5d..0000000 --- a/backend/scripts/target_index/parsers/eitc_state.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Parser for storage/calibration_targets/eitc_state.csv. - -IRS SOI Historical Table 2, EITC by state (TY 2022). The first line is a -`#`-prefixed citation comment; the next line is the header -`GEO_ID,Returns,Amount`. - -Each row yields TWO target records at state level: -- `tax_unit_count` (count) constrained by `eitc > 0` and filer gate — the - Returns column counts EITC-claiming filer tax units. -- `eitc` dollar amount. - -Both targets are filer-gated (`tax_unit_is_filer == 1`). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/eitc_state.csv" -PERIOD = 2022 - - -def _geo_id_to_state_fips(geo_id: str) -> str | None: - """0400000US01 → '01'. Returns None if not a state GEO_ID.""" - geo_id = geo_id.strip() - if geo_id.startswith("0400000US"): - return geo_id[len("0400000US"):] - return None - - -def _uncommented(lines): - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - yield line - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(_uncommented(f)) - for i, row in enumerate(reader): - state_fips = _geo_id_to_state_fips(row["GEO_ID"]) - if state_fips is None: - continue - returns = float(row["Returns"]) - amount = float(row["Amount"]) - row_key = f"{row['GEO_ID']}#{i}" - - # Filer-gated, EITC-claiming tax unit count. - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="state", - geographic_id=state_fips, - period=PERIOD, - constraints=( - ("tax_unit_is_filer", "==", "1"), - ("eitc", ">", "0"), - ), - value=returns, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="IRS SOI Historical Table 2 (TY2022) — EITC returns", - )) - # Dollar EITC amount, filer-gated. - out.append(TargetRecord( - variable="eitc", - geo_level="state", - geographic_id=state_fips, - period=PERIOD, - constraints=( - ("tax_unit_is_filer", "==", "1"), - ), - value=amount, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="IRS SOI Historical Table 2 (TY2022) — EITC amount", - )) - return out diff --git a/backend/scripts/target_index/parsers/healthcare_spending.py b/backend/scripts/target_index/parsers/healthcare_spending.py deleted file mode 100644 index 9a2abf6..0000000 --- a/backend/scripts/target_index/parsers/healthcare_spending.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Parser for storage/calibration_targets/healthcare_spending.csv. - -Schema (header has the column name `age_10_year_lower_bound` duplicated): - age_10_year_lower_bound, - health_insurance_premiums_without_medicare_part_b, - over_the_counter_health_expenses, - other_medical_expenses, - medicare_part_b_premiums, - age, - age_10_year_lower_bound - -10 rows, one per 10-year age band (0, 10, 20, …, 80). - -Each row contributes FOUR national-level $-amount targets (period=2024), -one per dollar variable. The age band is encoded as a pair of constraints -(`age >= lower`, `age < lower + 10`); the top band (80+) drops the upper -bound and uses only `age >= 80`. - -We deliberately use csv.reader rather than DictReader because the header -contains a duplicated column name (DictReader would collapse them and we'd -lose the first occurrence). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/healthcare_spending.csv" -PERIOD = 2024 -BAND_WIDTH = 10 -TOP_BAND_LOWER = 80 # 80+ is open-ended; no upper bound applied. - -# Column positions in the CSV row (0-indexed). -# Although the header has `age_10_year_lower_bound` in BOTH position 0 and -# position 6, inspection of the data shows column 0 holds the clean integer -# band lower bound (0, 10, 20, …, 80) while column 6 holds an aggregated -# dollar amount (e.g. 1.08e9 for the 80+ band) — *not* the band lower bound. -# We therefore use column 0 as the canonical lower bound. (The spec hint -# suggested the second occurrence; the data overrides the hint here.) -COL_BAND_LOWER = 0 -COL_HIP = 1 # health_insurance_premiums_without_medicare_part_b -COL_OTC = 2 # over_the_counter_health_expenses -COL_OTHER = 3 # other_medical_expenses -COL_MEDB = 4 # medicare_part_b_premiums -# COL_AGE = 5 — mean age within band, unused for target construction -# Column 6 is a duplicated header but holds a separate aggregate; unused. - -DOLLAR_VARS = [ - ("health_insurance_premiums_without_medicare_part_b", COL_HIP), - ("over_the_counter_health_expenses", COL_OTC), - ("other_medical_expenses", COL_OTHER), - ("medicare_part_b_premiums", COL_MEDB), -] - - -def _band_constraints(lower: float) -> tuple[tuple[str, str, str], ...]: - cons: list[tuple[str, str, str]] = [("age", ">=", str(int(lower)))] - if int(lower) < TOP_BAND_LOWER: - cons.append(("age", "<", str(int(lower) + BAND_WIDTH))) - return tuple(cons) - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.reader(f) - next(reader, None) # skip the (duplicate-column) header - for i, row in enumerate(reader): - if not row: - continue - lower = float(row[COL_BAND_LOWER]) - cons = _band_constraints(lower) - for var, col in DOLLAR_VARS: - value = float(row[col]) - out.append(TargetRecord( - variable=var, - geo_level="national", - geographic_id=None, - period=PERIOD, - constraints=cons, - value=value, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"row-{i + 2}/band-{int(lower)}", - notes=f"Healthcare spending {PERIOD} · age band {int(lower)}+", - )) - return out diff --git a/backend/scripts/target_index/parsers/medicaid_enrollment_2024.py b/backend/scripts/target_index/parsers/medicaid_enrollment_2024.py deleted file mode 100644 index 8b12af1..0000000 --- a/backend/scripts/target_index/parsers/medicaid_enrollment_2024.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Parser for storage/calibration_targets/medicaid_enrollment_2024.csv. - -Schema: state, enrollment — 51 rows (50 states + DC). -Each row → one TargetRecord: person-level count of Medicaid enrollees, at -state level, period=2024, constrained by `medicaid_enrolled == 1`. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/medicaid_enrollment_2024.csv" -PERIOD = 2024 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip() - fips = STATE_CODE_TO_FIPS.get(state) - if fips is None: - continue - enrollment = float(row["enrollment"]) - out.append(TargetRecord( - variable="person_count", - geo_level="state", - geographic_id=fips, - period=PERIOD, - constraints=(("medicaid_enrolled", "==", "1"),), - value=enrollment, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"row-{i + 2}", - notes=f"Medicaid enrollment {PERIOD} — {state} (persons with medicaid_enrolled==1)", - )) - return out diff --git a/backend/scripts/target_index/parsers/medicaid_enrollment_2025.py b/backend/scripts/target_index/parsers/medicaid_enrollment_2025.py deleted file mode 100644 index f72407e..0000000 --- a/backend/scripts/target_index/parsers/medicaid_enrollment_2025.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Parser for storage/calibration_targets/medicaid_enrollment_2025.csv. - -Same schema as medicaid_enrollment_2024.csv; period=2025. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/medicaid_enrollment_2025.csv" -PERIOD = 2025 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip() - fips = STATE_CODE_TO_FIPS.get(state) - if fips is None: - continue - enrollment = float(row["enrollment"]) - out.append(TargetRecord( - variable="person_count", - geo_level="state", - geographic_id=fips, - period=PERIOD, - constraints=(("medicaid_enrolled", "==", "1"),), - value=enrollment, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"row-{i + 2}", - notes=f"Medicaid enrollment {PERIOD} — {state} (persons with medicaid_enrolled==1)", - )) - return out diff --git a/backend/scripts/target_index/parsers/medicaid_enrollment_2026.py b/backend/scripts/target_index/parsers/medicaid_enrollment_2026.py deleted file mode 100644 index b22412f..0000000 --- a/backend/scripts/target_index/parsers/medicaid_enrollment_2026.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Parser for storage/calibration_targets/medicaid_enrollment_2026.csv. - -Same schema as medicaid_enrollment_2024.csv; period=2026. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/medicaid_enrollment_2026.csv" -PERIOD = 2026 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip() - fips = STATE_CODE_TO_FIPS.get(state) - if fips is None: - continue - enrollment = float(row["enrollment"]) - out.append(TargetRecord( - variable="person_count", - geo_level="state", - geographic_id=fips, - period=PERIOD, - constraints=(("medicaid_enrolled", "==", "1"),), - value=enrollment, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"row-{i + 2}", - notes=f"Medicaid enrollment {PERIOD} — {state} (persons with medicaid_enrolled==1)", - )) - return out diff --git a/backend/scripts/target_index/parsers/np2023_d5_mid.py b/backend/scripts/target_index/parsers/np2023_d5_mid.py deleted file mode 100644 index a1b7d69..0000000 --- a/backend/scripts/target_index/parsers/np2023_d5_mid.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Parser for storage/calibration_targets/np2023_d5_mid.csv. - -Census National Population Projections (2023 vintage, middle series). -Schema (wide, 2581 rows): - NATIVITY, RACE_HISP, SEX, YEAR, TOTAL_POP, POP_0, POP_1, ..., POP_85 - -Schema decision (pragmatic — see brief): the CSV is dimensioned by nativity -× race/hispanic origin × sex × year × age. We only ingest the "all -categories" rows so the audit doesn't drown in millions of disaggregated -cells. - -Census's own "all categories" coding uses 0 for RACE_HISP and SEX (those -columns have a literal 0 row meaning "Total"). NATIVITY, however, has no -"0" row in this file — only 1 (Total/native projection series) and 2 -(foreign-born). The NATIVITY=1 row reports a 2024 total of ~288.9M, which -matches the published US resident-population projection, so we treat -NATIVITY=1 as the "all" series and ingest those rows. - -Each kept row (one per YEAR) → 87 targets: - - 1 TOTAL_POP target per year (national, no age constraint). - - 86 per-age targets, one per POP_N column, constrained by age == N. - -If no NATIVITY=1 rows are present we fall back to ingesting nothing for -that year; in practice every projection year has one. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/np2023_d5_mid.csv" - -# NATIVITY=1 acts as the "all" series (NATIVITY=0 is not present in this file). -ALL_NATIVITY = "1" -ALL_RACE_HISP = "0" -ALL_SEX = "0" - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - pop_cols = [c for c in (reader.fieldnames or []) if c.startswith("POP_")] - # POP_N → age N (POP_85 is the open-ended top group; we still encode it - # as age == 85 to mirror how the DB strata key the projection). - age_for_col = {c: int(c.split("_", 1)[1]) for c in pop_cols} - - for i, row in enumerate(reader): - if (row["NATIVITY"].strip() != ALL_NATIVITY - or row["RACE_HISP"].strip() != ALL_RACE_HISP - or row["SEX"].strip() != ALL_SEX): - continue - - year = int(row["YEAR"]) - row_key = f"row-{i + 2}/year-{year}" - - # Total population for the year (no age constraint). - try: - total = float(row["TOTAL_POP"]) - except (TypeError, ValueError): - total = None - if total is not None: - out.append(TargetRecord( - variable="person_count", - geo_level="national", - geographic_id=None, - period=year, - value=total, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"{row_key}/total", - notes="Census NP2023 mid-series — total US population", - )) - - # Per-age targets. - for col in pop_cols: - raw = (row.get(col) or "").strip() - if not raw: - continue - try: - value = float(raw) - except ValueError: - continue - age = age_for_col[col] - out.append(TargetRecord( - variable="person_count", - geo_level="national", - geographic_id=None, - period=year, - constraints=(("age", "==", str(age)),), - value=value, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"{row_key}/{col}", - notes=f"Census NP2023 mid-series — age {age}", - )) - return out diff --git a/backend/scripts/target_index/parsers/population_by_state.py b/backend/scripts/target_index/parsers/population_by_state.py deleted file mode 100644 index ad8474f..0000000 --- a/backend/scripts/target_index/parsers/population_by_state.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Parser for storage/calibration_targets/population_by_state.csv. - -Schema: state, population, population_under_5 — 51 rows (state abbreviations). - -Each row → TWO targets: - 1. person_count, state-level total (no constraints). - 2. person_count, state-level, constrained by age < 5. - -Both are counts. period=2024 (undated; mirrors DB period family). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/population_by_state.csv" -PERIOD = 2024 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state"].strip().upper() - fips = STATE_CODE_TO_FIPS.get(state) - if fips is None: - continue - total = float(row["population"]) - under_5 = float(row["population_under_5"]) - row_key = f"row-{i + 2}" - - out.append(TargetRecord( - variable="person_count", - geo_level="state", - geographic_id=fips, - period=PERIOD, - value=total, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"{row_key}/total", - notes="Census state population — total", - )) - out.append(TargetRecord( - variable="person_count", - geo_level="state", - geographic_id=fips, - period=PERIOD, - constraints=(("age", "<", "5"),), - value=under_5, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"{row_key}/under_5", - notes="Census state population — under 5", - )) - return out diff --git a/backend/scripts/target_index/parsers/real_estate_taxes_by_state_acs.py b/backend/scripts/target_index/parsers/real_estate_taxes_by_state_acs.py deleted file mode 100644 index aa172fd..0000000 --- a/backend/scripts/target_index/parsers/real_estate_taxes_by_state_acs.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Parser for storage/calibration_targets/real_estate_taxes_by_state_acs.csv. - -Schema: state_code, real_estate_taxes_bn — 51 rows. The `_bn` suffix means -the value is in BILLIONS of dollars; multiply by 1e9 for the canonical -target value (which must be in dollars to match the DB). - -Each row → ONE target: - variable = real_estate_taxes - geo_level = state - geographic_id = state_fips (from STATE_CODE_TO_FIPS lookup) - value = real_estate_taxes_bn * 1e9 - is_count = False - period = 2024 -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/real_estate_taxes_by_state_acs.csv" -PERIOD = 2024 - -STATE_CODE_TO_FIPS = { - "AL": "1", "AK": "2", "AZ": "4", "AR": "5", "CA": "6", "CO": "8", - "CT": "9", "DE": "10", "DC": "11", "FL": "12", "GA": "13", "HI": "15", - "ID": "16", "IL": "17", "IN": "18", "IA": "19", "KS": "20", "KY": "21", - "LA": "22", "ME": "23", "MD": "24", "MA": "25", "MI": "26", "MN": "27", - "MS": "28", "MO": "29", "MT": "30", "NE": "31", "NV": "32", "NH": "33", - "NJ": "34", "NM": "35", "NY": "36", "NC": "37", "ND": "38", "OH": "39", - "OK": "40", "OR": "41", "PA": "42", "RI": "44", "SC": "45", "SD": "46", - "TN": "47", "TX": "48", "UT": "49", "VT": "50", "VA": "51", "WA": "53", - "WV": "54", "WI": "55", "WY": "56", -} - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - state = row["state_code"].strip().upper() - fips = STATE_CODE_TO_FIPS.get(state) - if fips is None: - continue - value_bn = float(row["real_estate_taxes_bn"]) - value = value_bn * 1e9 - - out.append(TargetRecord( - variable="real_estate_taxes", - geo_level="state", - geographic_id=fips, - period=PERIOD, - value=value, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=f"row-{i + 2}", - notes="ACS — real estate taxes by state ($, scaled ×1e9 from billions)", - )) - return out diff --git a/backend/scripts/target_index/parsers/snap_state.py b/backend/scripts/target_index/parsers/snap_state.py deleted file mode 100644 index 5a892bd..0000000 --- a/backend/scripts/target_index/parsers/snap_state.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Parser for storage/calibration_targets/snap_state.csv. - -Schema: GEO_ID, Households, Cost — 52 rows (DC + states + national row). -Each row produces TWO target records: a `snap` cost (in dollars) and a -`snap_unit` enrollment count, both at state level (or national for GEO_ID -matching 0100000US). - -Period is 2024 (the file is undated; the DB-side period is 2024 for this -variable family, so we mirror it). -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -SOURCE_PATH = "storage/calibration_targets/snap_state.csv" -PERIOD = 2024 - - -def _geo_id_to_geo(geo_id: str) -> tuple[str, str | None]: - """0400000US01 → ('state', '01'). 0100000US → ('national', None).""" - geo_id = geo_id.strip() - if geo_id.startswith("0400000US"): - state_fips = geo_id[len("0400000US"):] - return "state", state_fips - if geo_id.startswith("0100000US"): - return "national", None - return "national", None - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(_uncommented(f)) - for i, row in enumerate(reader): - geo_level, gid = _geo_id_to_geo(row["GEO_ID"]) - households = float(row["Households"]) - cost = float(row["Cost"]) - row_key = f"row-{i + 2}" # +2 for header + 1-indexed - - # The DB encodes the household-count target as variable=snap with - # constraint `snap > 0` (counts the population where snap > 0). - out.append(TargetRecord( - variable="snap", - geo_level=geo_level, - geographic_id=gid, - period=PERIOD, - constraints=(("snap", ">", "0"),), - value=households, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="USDA FNS SNAP — households enrolled (snap > 0 count)", - )) - out.append(TargetRecord( - variable="snap", - geo_level=geo_level, - geographic_id=gid, - period=PERIOD, - value=cost, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes="USDA FNS SNAP — dollar cost", - )) - return out - - -def _uncommented(lines): - """Yield only non-comment, non-empty lines (some CSVs start with a # header).""" - for line in lines: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - yield line diff --git a/backend/scripts/target_index/parsers/soi_targets.py b/backend/scripts/target_index/parsers/soi_targets.py deleted file mode 100644 index ac5d9f6..0000000 --- a/backend/scripts/target_index/parsers/soi_targets.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Parser for storage/calibration_targets/soi_targets.csv. - -The largest CSV (~11.9k rows). Each row is an IRS Statistics of Income target: -- Year: e.g. 2015, 2018, 2022 -- SOI table: e.g. "Table 1.1" -- XLSX column / row: spreadsheet coords (used as part of source_row) -- Variable: PE variable name (adjusted_gross_income, eitc, etc.) -- Filing status: All / Joint / Single / HoH / MFS / SeparateReturns -- AGI lower/upper bound: numeric (with -inf / inf for open-ended) -- Count flag: True → row is a return-count target; False → dollar amount -- Taxable only: filter narrowing the population -- Full population: another filter flag -- Value: the actual target number - -These are all national-level (no state breakdown) but heavily constrained -by AGI band + filing status + filer-only restrictions. -""" - -from __future__ import annotations - -import csv -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/soi_targets.csv" - - -def _parse_bound(s: str) -> float: - s = s.strip() - if s in ("-inf", "-Infinity"): return float("-inf") - if s in ("inf", "Infinity"): return float("inf") - return float(s) - - -def _constraints_from_row(row: dict) -> tuple[tuple[str, str, str], ...]: - """Build the constraint tuple that mirrors how the pipeline strata are - keyed in policy_data.db. Order matters for canonical signature matching.""" - cons: list[tuple[str, str, str]] = [] - - # Filer gating: all SOI targets implicitly assume tax_unit_is_filer == 1 - cons.append(("tax_unit_is_filer", "==", "1")) - - # Filing status (when not "All") - fs = row["Filing status"].strip() - if fs and fs.lower() != "all": - cons.append(("tax_unit_filing_status", "==", fs)) - - # AGI band - lo = _parse_bound(row["AGI lower bound"]) - hi = _parse_bound(row["AGI upper bound"]) - if lo != float("-inf"): - cons.append(("adjusted_gross_income", ">=", str(lo))) - if hi != float("inf"): - cons.append(("adjusted_gross_income", "<", str(hi))) - - # Taxable-only / Full-population flags - if _truthy(row.get("Taxable only")): - cons.append(("tax_unit_is_taxable", "==", "1")) - if _truthy(row.get("Full population")): - cons.append(("full_population", "==", "1")) - - return tuple(cons) - - -def _truthy(val) -> bool: - if isinstance(val, bool): return val - if val is None: return False - s = str(val).strip().lower() - return s in ("true", "1", "yes") - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - try: - year = int(row["Year"]) - value = float(row["Value"]) - except (TypeError, ValueError): - continue # malformed row, skip silently — audit will see it missing - - variable = row["Variable"].strip() - is_count = _truthy(row.get("Count")) - - cons = _constraints_from_row(row) - row_key = f"{row['SOI table']}/{row['XLSX row']}/{row['XLSX column']}" - - out.append(TargetRecord( - variable=variable, - geo_level="national", - geographic_id=None, - period=year, - constraints=cons, - value=value, - is_count=is_count, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"IRS SOI {row['SOI table']} · filing={row['Filing status']}", - )) - return out diff --git a/backend/scripts/target_index/parsers/spm_threshold_agi.py b/backend/scripts/target_index/parsers/spm_threshold_agi.py deleted file mode 100644 index 87b5b4e..0000000 --- a/backend/scripts/target_index/parsers/spm_threshold_agi.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Parser for storage/calibration_targets/spm_threshold_agi.csv. - -Schema: decile, lower_spm_threshold, upper_spm_threshold, - adjusted_gross_income, count - -11 rows (one per SPM-threshold decile; the top row has upper=inf). -Each row → TWO national-level TargetRecords (period=2024 — file is undated): - * adjusted_gross_income dollar amount - * tax_unit_count count - -The SPM-threshold decile bracket is encoded as constraints on -`spm_unit_spm_threshold` (>= lower, < upper) — except the open-ended top -band, which drops the upper bound. -""" - -from __future__ import annotations - -import csv -import math -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - -SOURCE_PATH = "storage/calibration_targets/spm_threshold_agi.csv" -PERIOD = 2024 - - -def _parse_bound(s: str) -> float: - s = s.strip() - if s in ("inf", "Infinity"): - return float("inf") - if s in ("-inf", "-Infinity"): - return float("-inf") - return float(s) - - -def _bracket_constraints(lower: float, upper: float) -> tuple[tuple[str, str, str], ...]: - cons: list[tuple[str, str, str]] = [] - if math.isfinite(lower): - cons.append(("spm_unit_spm_threshold", ">=", str(lower))) - if math.isfinite(upper): - cons.append(("spm_unit_spm_threshold", "<", str(upper))) - return tuple(cons) - - -def parse(csv_path: Path) -> list[TargetRecord]: - out: list[TargetRecord] = [] - with csv_path.open() as f: - reader = csv.DictReader(f) - for i, row in enumerate(reader): - decile = row["decile"].strip() - lower = _parse_bound(row["lower_spm_threshold"]) - upper = _parse_bound(row["upper_spm_threshold"]) - agi = float(row["adjusted_gross_income"]) - count = float(row["count"]) - cons = _bracket_constraints(lower, upper) - row_key = f"row-{i + 2}/decile-{decile}" - - out.append(TargetRecord( - variable="adjusted_gross_income", - geo_level="national", - geographic_id=None, - period=PERIOD, - constraints=cons, - value=agi, - is_count=False, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"SPM-threshold decile {decile} · AGI dollar amount", - )) - out.append(TargetRecord( - variable="tax_unit_count", - geo_level="national", - geographic_id=None, - period=PERIOD, - constraints=cons, - value=count, - is_count=True, - storage_tier="csv", - source_path=SOURCE_PATH, - source_row=row_key, - notes=f"SPM-threshold decile {decile} · tax-unit count", - )) - return out diff --git a/backend/scripts/target_index/python_constants.py b/backend/scripts/target_index/python_constants.py deleted file mode 100644 index bf5df9a..0000000 --- a/backend/scripts/target_index/python_constants.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Tier 3 reader: hand-coded target constants in policyengine_us_data.utils. - -Returns TargetRecords for the dicts the calibration team maintains as -Python literals (not CSVs). Today these cover Medicare (4 dicts) and a -post-calibration ACA take-up target. -""" - -from __future__ import annotations - -from pathlib import Path - -from backend.scripts.target_index.schema import TargetRecord - - -def collect() -> list[TargetRecord]: - out: list[TargetRecord] = [] - - # --- Medicare (utils/cms_medicare.py) ---------------------------------- - try: - from policyengine_us_data.utils import cms_medicare as cms - except ImportError: - cms = None - if cms is not None: - src = "utils/cms_medicare.py" - - for year, value in cms.MEDICARE_PART_B_GROSS_PREMIUM_INCOME.items(): - out.append(TargetRecord( - variable="medicare_part_b", # gross premium dollars - geo_level="national", - period=year, - value=float(value), - is_count=False, - storage_tier="python", - source_path=src, - source_row="MEDICARE_PART_B_GROSS_PREMIUM_INCOME", - notes="CMS Medicare Trustees Report — gross Part B premium income", - )) - - for year, value in cms.MEDICARE_ENROLLMENT_TARGETS.items(): - out.append(TargetRecord( - variable="person_count", - geo_level="national", - period=year, - constraints=(("medicare_enrolled", "==", "1"),), - value=float(value), - is_count=True, - storage_tier="python", - source_path=src, - source_row="MEDICARE_ENROLLMENT_TARGETS", - notes="CMS Medicare Trustees Report Table V.B3 — enrollee count", - )) - - for year, value in cms.MEDICARE_STATE_BUY_IN_MINIMUM_BENEFICIARIES.items(): - out.append(TargetRecord( - variable="person_count", - geo_level="national", - period=year, - constraints=(("state_buy_in_medicare", "==", "1"),), - value=float(value), - is_count=True, - storage_tier="python", - source_path=src, - source_row="MEDICARE_STATE_BUY_IN_MINIMUM_BENEFICIARIES", - notes="CMS state buy-in Medicare beneficiaries minimum", - )) - - for year, value in cms.BENEFICIARY_PAID_MEDICARE_PART_B_PREMIUM_TARGETS.items(): - out.append(TargetRecord( - variable="medicare_part_b_premiums", - geo_level="national", - period=year, - value=float(value), - is_count=False, - storage_tier="python", - source_path=src, - source_row="BENEFICIARY_PAID_MEDICARE_PART_B_PREMIUM_TARGETS", - notes="CMS — beneficiary-paid Medicare Part B premiums total", - )) - - # --- ACA post-calibration (utils/takeup.py) ---------------------------- - try: - from policyengine_us_data.utils import takeup - except ImportError: - takeup = None - if takeup is not None and hasattr(takeup, "ACA_POST_CALIBRATION_PERSON_TARGETS"): - src = "utils/takeup.py" - for year, value in takeup.ACA_POST_CALIBRATION_PERSON_TARGETS.items(): - out.append(TargetRecord( - variable="person_count", - geo_level="national", - period=year, - constraints=(("aca_ptc", ">", "0"),), - value=float(value), - is_count=True, - storage_tier="python", - source_path=src, - source_row="ACA_POST_CALIBRATION_PERSON_TARGETS", - notes="CMS Marketplace OEP — APTC consumers (post-calibration fallback)", - )) - - return out diff --git a/backend/scripts/target_index/schema.py b/backend/scripts/target_index/schema.py deleted file mode 100644 index 4cc9b48..0000000 --- a/backend/scripts/target_index/schema.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Unified target record schema spanning the 5 storage tiers.""" - -from __future__ import annotations - -from dataclasses import dataclass, field, asdict -from typing import Literal - - -StorageTier = Literal["db", "csv", "python", "generator", "yaml"] - - -@dataclass -class TargetRecord: - """One calibration target, normalised across all source tiers. - - The `signature` is what we use to match the same logical target across - tiers — e.g. tier-1 CSV row for SNAP-Alabama-2024 should signature-match - the tier-5 DB row produced from it. - """ - - # Identity (used for matching) - variable: str - geo_level: str | None = None # "national" | "state" | "district" | None - geographic_id: str | None = None # e.g. "01" (state fips) or "0612" (CD) - period: int | None = None - constraints: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) - # Each constraint is (variable, operation, value) — e.g. ("tax_unit_is_filer", "==", "1") - - # Value - value: float | None = None - is_count: bool = False # True when value is a count, not a $ amount - - # Provenance - storage_tier: StorageTier = "csv" - source_path: str = "" # e.g. "storage/calibration_targets/snap_state.csv" - source_row: str = "" # row index, line number, or symbolic key - notes: str = "" # free text — usually the data team's citation - - def signature(self) -> tuple: - """Canonical key for cross-tier matching. - - - geographic_id is normalised through int() when numeric so '01' and - '1' compare equal (CSVs use Census GEO_ID strings, the DB stores - state_fips as bare integers). - - constraint values are normalised the same way for the same reason. - """ - return ( - self.variable, - self.geo_level or "", - _norm_geo_id(self.geographic_id), - self.period or 0, - tuple(sorted((c[0], c[1], _norm_geo_id(c[2])) for c in self.constraints)), - ) - - def to_dict(self) -> dict: - d = asdict(self) - d["constraints"] = [list(c) for c in self.constraints] - d["signature"] = list(self.signature()) - return d - - -def _norm_geo_id(val) -> str: - """Normalise a geographic id / constraint value: strip leading zeros for - numeric strings; otherwise keep as-is.""" - if val is None or val == "": - return "" - s = str(val) - try: - f = float(s) - except (TypeError, ValueError): - return s - # Preserve inf/nan as their string form so AGI band bounds remain comparable. - import math - if not math.isfinite(f): - return s - return str(int(f)) diff --git a/backend/services/__init__.py b/backend/services/__init__.py deleted file mode 100644 index 436b05b..0000000 --- a/backend/services/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Service modules.""" diff --git a/backend/services/analysis_readiness.py b/backend/services/analysis_readiness.py deleted file mode 100644 index eef87c3..0000000 --- a/backend/services/analysis_readiness.py +++ /dev/null @@ -1,1131 +0,0 @@ -"""Analyst-facing readiness checks for reform analysis. - -This module turns the lower-level calibration artifacts into answers an -analyst can use before trusting a reform estimate: - -- Are the relevant aggregates targeted? -- Were those targets included in the fitted loss? -- Did the published dataset evaluate them successfully? -- Which upstream nodes feed a selected output, and which leaves are only - carried by source data rather than calibration targets? -""" - -from __future__ import annotations - -from collections import deque -from dataclasses import dataclass -from typing import Any - -import numpy as np -import pandas as pd - -from backend.state import AppState - - -@dataclass(frozen=True) -class CaseStudy: - id: str - label: str - description: str - primary_variables: tuple[str, ...] - dependency_variables: tuple[str, ...] - state_fips: int | None = None - domain_keywords: tuple[str, ...] = () - modeled_variable_prefixes: tuple[str, ...] = () - - -CASE_STUDIES: dict[str, CaseStudy] = { - "federal_snap": CaseStudy( - id="federal_snap", - label="Federal SNAP reform", - description=( - "Assess whether SNAP benefit and caseload aggregates are usable " - "for a federal reform analysis." - ), - primary_variables=("snap", "household_count"), - dependency_variables=("snap",), - domain_keywords=("snap",), - ), - "montana_ctc": CaseStudy( - id="montana_ctc", - label="Montana CTC reform", - description=( - "Assess whether Montana child tax credit analysis is supported by " - "state-level CTC targets and modeled PolicyEngine variables." - ), - primary_variables=( - "ctc", - "refundable_ctc", - "non_refundable_ctc", - "tax_unit_count", - ), - dependency_variables=("ctc", "refundable_ctc", "non_refundable_ctc"), - state_fips=30, - domain_keywords=("ctc", "refundable_ctc", "non_refundable_ctc"), - modeled_variable_prefixes=("mt_ctc", "mt_refundable_ctc"), - ), - "california_income_tax": CaseStudy( - id="california_income_tax", - label="California income tax reform", - description=( - "Assess whether California income tax analysis is supported by " - "state income-tax targets and modeled California tax variables." - ), - primary_variables=( - "ca_income_tax", - "income_tax", - "state_income_tax", - "tax_unit_count", - ), - dependency_variables=( - "ca_income_tax", - "ca_income_tax_before_credits", - "ca_income_tax_before_refundable_credits", - ), - state_fips=6, - domain_keywords=("income_tax", "ca_income_tax", "state_income_tax"), - modeled_variable_prefixes=("ca_income_tax",), - ), -} - - -def list_case_studies() -> list[dict[str, Any]]: - return [ - { - "id": cs.id, - "label": cs.label, - "description": cs.description, - "primary_variables": list(cs.primary_variables), - "dependency_variables": list(cs.dependency_variables), - "state_fips": cs.state_fips, - } - for cs in CASE_STUDIES.values() - ] - - -def list_policyengine_variables( - state: AppState, - *, - search: str | None = None, - limit: int = 100, -) -> list[dict[str, Any]]: - """Return every PolicyEngine variable available in the loaded simulation.""" - tbs = _tax_benefit_system(state) - if tbs is None: - return [] - - targets = state.targets_enriched - target_counts: dict[str, int] = {} - included_counts: dict[str, int] = {} - domain_counts: dict[str, int] = {} - if targets is not None and not targets.empty: - variable_col = targets.get("variable", pd.Series([], dtype=str)).fillna("") - target_counts = variable_col.astype(str).value_counts().to_dict() - if "included" in targets.columns: - included = targets[targets["included"].astype(bool)] - included_counts = ( - included.get("variable", pd.Series([], dtype=str)) - .fillna("") - .astype(str) - .value_counts() - .to_dict() - ) - domain_col = targets.get("domain_variable", pd.Series("", index=targets.index)) - for value in domain_col.fillna("").astype(str): - for part in [p.strip() for p in value.split(",") if p.strip()]: - domain_counts[part] = domain_counts.get(part, 0) + 1 - - items = [] - for name, var in getattr(tbs, "variables", {}).items(): - label = getattr(var, "label", None) or name - if search: - q = search.lower() - if q not in name.lower() and q not in label.lower(): - continue - items.append( - { - "name": name, - "label": label, - "entity": getattr(getattr(var, "entity", None), "key", None), - "definition_period": str(getattr(var, "definition_period", "") or ""), - "is_formula": bool(getattr(var, "formulas", None)), - "is_aggregate": bool( - getattr(var, "adds", None) or getattr(var, "subtracts", None) - ), - "is_target_variable": target_counts.get(name, 0) > 0, - "is_domain_variable": domain_counts.get(name, 0) > 0, - "target_count": int(target_counts.get(name, 0)), - "included_target_count": int(included_counts.get(name, 0)), - "domain_count": int(domain_counts.get(name, 0)), - } - ) - items.sort( - key=lambda r: ( - 0 if r["is_target_variable"] or r["is_domain_variable"] else 1, - r["name"], - ) - ) - return items[:limit] - - -def _sim_from_state(state: AppState): - sim = state.sim_service - return getattr(sim, "_sim", sim) - - -def _tax_benefit_system(state: AppState): - sim = _sim_from_state(state) - return getattr(sim, "tax_benefit_system", None) - - -def _available_model_variables(state: AppState) -> set[str]: - tbs = _tax_benefit_system(state) - if tbs is None: - return set() - return set(getattr(tbs, "variables", {}).keys()) - - -def _fallback_target_config() -> tuple[dict | None, str | None]: - try: - import yaml - import policyengine_us_data - - from pathlib import Path - - root = Path(policyengine_us_data.__file__).resolve().parent - path = root / "calibration" / "target_config.yaml" - if not path.exists(): - return None, None - config = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - return config, f"installed_package:{path}" - except Exception: - return None, None - - -def _target_config_with_source(state: AppState) -> tuple[dict | None, str]: - if state.target_config: - return state.target_config, "run_artifact" - config, source = _fallback_target_config() - if config: - return config, source or "installed_package" - return None, "unavailable" - - -def _targets_with_inclusion(state: AppState) -> pd.DataFrame: - targets = state.targets_enriched.copy() - if targets.empty: - return targets - if not targets.get("included", pd.Series(False, index=targets.index)).any(): - config, _source = _target_config_with_source(state) - if config: - targets["included"] = _infer_included_from_config(targets, config) - return targets - - -def _match_config_rules(targets: pd.DataFrame, rules: list[dict]) -> pd.Series: - mask = pd.Series(False, index=targets.index) - for rule in rules: - mask |= _target_config_rule_mask(targets, rule) - return mask - - -def _target_config_rule_mask(targets: pd.DataFrame, rule: dict) -> pd.Series: - mask = pd.Series(True, index=targets.index) - if "variable" in rule: - mask &= targets["variable"].astype(str) == str(rule["variable"]) - if "geo_level" in rule: - mask &= targets["geo_level"].astype(str) == str(rule["geo_level"]) - if "domain_variable" in rule: - domain = targets.get("domain_variable", pd.Series("", index=targets.index)) - mask &= domain.fillna("").astype(str) == str(rule["domain_variable"]) - return mask - - -def _target_rows_for_variable(targets: pd.DataFrame, variable: str) -> pd.Series: - if targets.empty: - return pd.Series(False, index=targets.index) - variable_col = targets.get("variable", pd.Series("", index=targets.index)) - direct = variable_col.fillna("").astype(str) == variable - domain_col = targets.get("domain_variable", pd.Series("", index=targets.index)) - domain = domain_col.fillna("").astype(str) - as_domain = domain.str.split(",").apply( - lambda values: variable in {part.strip() for part in values if part.strip()} - ) - return direct | as_domain - - -def _rule_mentions_variable(rule: dict, variable: str) -> bool: - if str(rule.get("variable", "")) == variable: - return True - domain = str(rule.get("domain_variable", "")) - return variable in {part.strip() for part in domain.split(",") if part.strip()} - - -def _infer_included_from_config(targets: pd.DataFrame, config: dict) -> pd.Series: - include_rules = config.get("include", []) - exclude_rules = config.get("exclude", []) - if include_rules: - keep = _match_config_rules(targets, include_rules) - else: - keep = pd.Series(True, index=targets.index) - if exclude_rules: - keep &= ~_match_config_rules(targets, exclude_rules) - return keep - - -def _target_mask(df: pd.DataFrame, case: CaseStudy) -> pd.Series: - variable = df.get("variable", pd.Series([], dtype=str)).fillna("").astype(str) - domain = df.get("domain_variable", pd.Series("", index=df.index)).fillna("") - domain = domain.astype(str) - - direct_variables = [ - v - for v in case.primary_variables - if v not in {"person_count", "household_count", "tax_unit_count"} - and not v.endswith("_count") - ] - mask = variable.isin(direct_variables) - for keyword in case.domain_keywords: - mask = mask | domain.str.contains(keyword, case=False, regex=False) - - if case.state_fips is not None: - gid = df.get("geographic_id", pd.Series("", index=df.index)).fillna("") - geo = df.get("geo_level", pd.Series("", index=df.index)).fillna("") - - def _state_match(value, geo_level) -> bool: - if geo_level == "national": - return True - s = str(value) - if not s.isdigit(): - return False - n = int(s) - return n == case.state_fips or n // 100 == case.state_fips - - geo_mask = pd.Series( - [_state_match(g, level) for g, level in zip(gid, geo)], - index=df.index, - ) - mask = mask & geo_mask - return mask - - -def _finite_float(value) -> float | None: - if value is None: - return None - try: - f = float(value) - except (TypeError, ValueError): - return None - return f if np.isfinite(f) else None - - -def _safe_bool(value) -> bool: - try: - if pd.isna(value): - return False - except (TypeError, ValueError): - pass - return bool(value) - - -def _error_summary(df: pd.DataFrame) -> dict[str, Any]: - if df.empty: - return { - "count": 0, - "evaluated": 0, - "median_abs_rel_error": None, - "max_abs_rel_error": None, - "pct_under_10pct": None, - "pct_under_25pct": None, - } - err = pd.to_numeric(df.get("abs_rel_error"), errors="coerce") - finite = err[np.isfinite(err)] - if finite.empty: - return { - "count": int(len(df)), - "evaluated": 0, - "median_abs_rel_error": None, - "max_abs_rel_error": None, - "pct_under_10pct": None, - "pct_under_25pct": None, - } - return { - "count": int(len(df)), - "evaluated": int(len(finite)), - "median_abs_rel_error": float(finite.median()), - "max_abs_rel_error": float(finite.max()), - "pct_under_10pct": float((finite < 0.10).mean()), - "pct_under_25pct": float((finite < 0.25).mean()), - } - - -def _bundle_summary(df: pd.DataFrame) -> dict[str, Any]: - err = pd.to_numeric(df.get("abs_rel_error"), errors="coerce") - finite = err[np.isfinite(err)] - included = df.get("included", pd.Series(False, index=df.index)).fillna(False).astype(bool) - if finite.empty: - return { - "target_count": int(len(df)), - "included_target_count": int(included.sum()), - "evaluated_target_count": 0, - "median_abs_rel_error": None, - "p90_abs_rel_error": None, - "max_abs_rel_error": None, - "pct_under_10pct": None, - "pct_under_25pct": None, - } - return { - "target_count": int(len(df)), - "included_target_count": int(included.sum()), - "evaluated_target_count": int(len(finite)), - "median_abs_rel_error": float(finite.median()), - "p90_abs_rel_error": float(finite.quantile(0.9)), - "max_abs_rel_error": float(finite.max()), - "pct_under_10pct": float((finite < 0.10).mean()), - "pct_under_25pct": float((finite < 0.25).mean()), - } - - -def _bundle_for_targets(state: AppState, dataset_file: str) -> pd.DataFrame: - from backend.services.geo_utils import runtime_dataset_bundle_for - - available = None - try: - from backend.services.runs import get_dataset - from backend.services.bundle_availability import published_bundles - - ds = get_dataset(state.dataset_id) - except Exception: - available = None - else: - if ds is not None: - available = published_bundles(ds.repo_id, state.run_id) - if available and dataset_file not in available: - raise KeyError(f"Dataset file is not published for this run: {dataset_file}") - - targets = state.targets_enriched.copy() - if targets.empty: - return targets - mask = targets.apply( - lambda row: runtime_dataset_bundle_for( - row.get("geo_level"), - row.get("geographic_id"), - available=available, - ) - == dataset_file, - axis=1, - ) - return targets[mask].copy() - - -def build_bundle_health( - state: AppState, - *, - dataset_file: str, - limit: int = 10, -) -> dict[str, Any]: - targets = _bundle_for_targets(state, dataset_file) - bundle_evaluated = False - evaluation_error: str | None = None - if not targets.empty and dataset_file != "enhanced_cps_2024.h5": - try: - from backend.services.runs import get_dataset - from backend.services.bundle_eval import evaluate_bundle - - ds = get_dataset(state.dataset_id) - if ds is not None: - targets = evaluate_bundle( - targets, - repo_id=ds.repo_id, - run_id=state.run_id, - bundle=dataset_file, - time_period=state.time_period, - ) - bundle_evaluated = True - except Exception as exc: - evaluation_error = str(exc) - - summary = _bundle_summary(targets) - variable_rows = [] - if not targets.empty: - for variable, group in targets.groupby("variable", dropna=False): - row = {"variable": str(variable), **_bundle_summary(group)} - variable_rows.append(row) - variable_rows.sort( - key=lambda row: ( - -1 if row["median_abs_rel_error"] is None else row["median_abs_rel_error"], - row["variable"], - ), - reverse=True, - ) - - if "abs_rel_error" in targets.columns: - worst_df = targets.copy() - worst_df["_sort_abs_rel_error"] = pd.to_numeric( - worst_df["abs_rel_error"], - errors="coerce", - ) - worst_df = worst_df.sort_values("_sort_abs_rel_error", ascending=False) - else: - worst_df = targets - - worst_targets = [] - for _, row in worst_df.head(limit).iterrows(): - target_value = _finite_float(row.get("value")) - estimate = _finite_float(row.get("estimate")) - worst_targets.append( - { - "target_id": int(row["target_id"]) if pd.notna(row.get("target_id")) else None, - "variable": str(row.get("variable", "")), - "geo_level": str(row.get("geo_level", "")), - "geographic_id": str(row.get("geographic_id", "")), - "target_value": target_value, - "estimate": estimate, - "rel_error": _finite_float(row.get("rel_error")), - "abs_rel_error": _finite_float(row.get("abs_rel_error")), - "included": _safe_bool(row.get("included")), - "source": str(row.get("source", "")) if pd.notna(row.get("source")) else None, - "constraints": _constraints_for_row(row), - } - ) - - return { - "dataset_file": dataset_file, - "bundle_evaluated": bundle_evaluated, - "evaluation_error": evaluation_error, - "summary": summary, - "by_variable": variable_rows[:50], - "worst_targets": worst_targets, - } - - -def _group_coverage(df: pd.DataFrame) -> list[dict[str, Any]]: - if df.empty: - return [] - group_cols = ["variable", "geo_level", "domain_variable"] - rows = [] - for keys, group in df.groupby(group_cols, dropna=False): - included = group[group.get("included", False).astype(bool)] - summary = _error_summary(included) - rows.append( - { - "variable": keys[0] or "", - "geo_level": keys[1] or "", - "domain_variable": keys[2] or "", - "target_count": int(len(group)), - "included_count": int(len(included)), - "evaluated_count": summary["evaluated"], - "median_abs_rel_error": summary["median_abs_rel_error"], - "max_abs_rel_error": summary["max_abs_rel_error"], - } - ) - rows.sort( - key=lambda r: ( - r["geo_level"], - r["variable"], - r["domain_variable"], - ) - ) - return rows - - -def _weight_quality(state: AppState, case: CaseStudy) -> dict[str, Any]: - households = state.households_df - if households is None or households.empty: - return { - "households": 0, - "kish_effective_n": None, - "top_1pct_weight_share": None, - "top_5pct_weight_share": None, - } - df = households - if case.state_fips is not None and "state" in df.columns: - df = df[df["state"].astype(int) == case.state_fips] - weights = pd.to_numeric(df.get("final_weight"), errors="coerce").fillna(0) - total = float(weights.sum()) - if len(weights) == 0 or total <= 0: - return { - "households": int(len(df)), - "kish_effective_n": None, - "top_1pct_weight_share": None, - "top_5pct_weight_share": None, - } - sorted_w = weights.sort_values(ascending=False).to_numpy() - top_1_n = max(1, int(np.ceil(len(sorted_w) * 0.01))) - top_5_n = max(1, int(np.ceil(len(sorted_w) * 0.05))) - denom = float(np.square(weights).sum()) - return { - "households": int(len(df)), - "kish_effective_n": float(total**2 / denom) if denom > 0 else None, - "top_1pct_weight_share": float(sorted_w[:top_1_n].sum() / total), - "top_5pct_weight_share": float(sorted_w[:top_5_n].sum() / total), - } - - -def _diagnosis( - *, - case: CaseStudy, - modeled_matches: list[str], - included: pd.DataFrame, - target_summary: dict[str, Any], - weight_quality: dict[str, Any], -) -> tuple[str, list[str], list[str]]: - blockers: list[str] = [] - warnings: list[str] = [] - - if case.modeled_variable_prefixes and not modeled_matches: - blockers.append( - "No existing PolicyEngine variable appears to model this state-specific reform." - ) - if included.empty: - blockers.append("No relevant targets are included in the calibration loss.") - elif target_summary["evaluated"] == 0: - blockers.append("Relevant in-loss targets have no available PE aggregate estimates.") - - max_err = target_summary.get("max_abs_rel_error") - med_err = target_summary.get("median_abs_rel_error") - if max_err is not None and max_err > 0.25: - warnings.append("At least one relevant in-loss target is more than 25% off.") - if med_err is not None and med_err > 0.10: - warnings.append("Median relevant target error is above 10%.") - if weight_quality.get("households") == 0: - warnings.append("No households are available for the selected geography.") - if _finite_float(weight_quality.get("top_1pct_weight_share")) is not None: - if weight_quality["top_1pct_weight_share"] > 0.25: - warnings.append("Top 1% of records carry more than 25% of weight.") - - if blockers: - status = "blocked" - elif warnings: - status = "caution" - else: - status = "ready" - return status, blockers, warnings - - -def build_readiness(case_id: str, state: AppState) -> dict[str, Any]: - if case_id not in CASE_STUDIES: - raise KeyError(case_id) - case = CASE_STUDIES[case_id] - target_config_source = "run_artifact" - inferred_inclusion = False - targets = state.targets_enriched.copy() - if not targets.empty and not targets.get("included", pd.Series(False, index=targets.index)).any(): - config, target_config_source = _target_config_with_source(state) - if config: - targets["included"] = _infer_included_from_config(targets, config) - inferred_inclusion = True - if targets.empty: - relevant = targets - else: - relevant = targets[_target_mask(targets, case)].copy() - included = relevant[relevant.get("included", False).astype(bool)].copy() - - model_vars = _available_model_variables(state) - modeled_matches = sorted( - v - for v in model_vars - if any(v.startswith(prefix) for prefix in case.modeled_variable_prefixes) - ) - present_dependency_variables = [ - v for v in case.dependency_variables if v in model_vars - ] - missing_dependency_variables = [ - v for v in case.dependency_variables if v not in model_vars - ] - - target_summary = _error_summary(included) - weight_quality = _weight_quality(state, case) - status, blockers, warnings = _diagnosis( - case=case, - modeled_matches=modeled_matches, - included=included, - target_summary=target_summary, - weight_quality=weight_quality, - ) - if inferred_inclusion: - warnings.append( - "Target inclusion is inferred from target_config because " - "published diagnostics were not available." - ) - if status == "ready": - status = "caution" - - recommendations = [] - if "federal_snap" == case.id: - recommendations.extend( - [ - "Compare national SNAP spending and state SNAP spending before interpreting reform costs.", - "Check SNAP household-count targets when analyzing caseload-sensitive reforms.", - "Use dependency tracing to review eligibility inputs such as household composition, income, and state rules.", - ] - ) - if "montana_ctc" == case.id: - recommendations.extend( - [ - "Add Montana-specific CTC variables and parameters in policyengine-us before treating this as a modeled current-law program.", - "Keep Montana refundable CTC targets visible, but add non-refundable/total CTC targets if the reform affects those components.", - "Run a Montana H5 bundle evaluation after changing CTC variables or target configuration.", - ] - ) - if "california_income_tax" == case.id: - recommendations.extend( - [ - "Trace ca_income_tax before changing parameters to confirm which income and credit nodes propagate.", - "Check California state-level income_tax targets and California district income-tax support before using local estimates.", - "Run the California state H5 bundle evaluation when validating aggregate impacts for CA-only reforms.", - ] - ) - if blockers: - recommendations.append("Resolve blockers before using this dataset for headline estimates.") - - return { - "case_study": { - "id": case.id, - "label": case.label, - "description": case.description, - "state_fips": case.state_fips, - "primary_variables": list(case.primary_variables), - "dependency_variables": list(case.dependency_variables), - }, - "status": status, - "blockers": blockers, - "warnings": warnings, - "target_summary": { - "relevant_targets": int(len(relevant)), - "included_targets": int(len(included)), - "inclusion_inferred": inferred_inclusion, - "target_config_source": target_config_source, - **target_summary, - }, - "target_coverage": _group_coverage(relevant), - "modeled_variables": { - "present_dependency_variables": present_dependency_variables, - "missing_dependency_variables": missing_dependency_variables, - "state_specific_matches": modeled_matches[:50], - }, - "weight_quality": weight_quality, - "recommendations": recommendations, - } - - -def _base_trace_key(key: str) -> str: - return key.split("<", 1)[0] - - -def _period_from_trace_key(key: str) -> str | None: - if "<" not in key or ">" not in key: - return None - return key.split("<", 1)[1].split(",", 1)[0].split(">", 1)[0] - - -def _parse_constraints_from_target_name(target_name: str) -> list[str]: - if not isinstance(target_name, str) or "[" not in target_name: - return [] - try: - bracket = target_name[target_name.index("[") + 1 : target_name.rindex("]")] - except ValueError: - return [] - if not bracket: - return [] - return [c.strip() for c in bracket.split(",") if c.strip()] - - -def _constraints_for_row(row: pd.Series) -> list[str]: - constraints = row.get("constraints") - if isinstance(constraints, list): - return [str(c) for c in constraints] - return _parse_constraints_from_target_name(str(row.get("target_name", ""))) - - -def _parse_constraint(constraint: str) -> tuple[str, str, str] | None: - for op in (">=", "<=", "==", "!=", ">", "<", "="): - if op in constraint: - left, right = constraint.split(op, 1) - return left.strip(), op, right.strip() - return None - - -def _float_bound(value: str) -> float | None: - try: - return float(value) - except (TypeError, ValueError): - lowered = str(value).strip().lower() - if lowered in {"inf", "+inf", "infinity", "+infinity"}: - return float("inf") - if lowered in {"-inf", "-infinity"}: - return float("-inf") - return None - - -def _format_bucket_bound(value: float | None) -> str: - if value is None: - return "?" - if value == float("-inf"): - return "-inf" - if value == float("inf"): - return "inf" - return f"{value:,.0f}" - - -def _bucket_from_constraints( - constraints: list[str], - domain_variable: str, -) -> tuple[float | None, float | None, str] | None: - lower: float | None = None - upper: float | None = None - equality: str | None = None - for constraint in constraints: - parsed = _parse_constraint(constraint) - if parsed is None: - continue - variable, op, value = parsed - if variable != domain_variable: - continue - bound = _float_bound(value) - if op in {">", ">="}: - lower = bound - if op in {"<", "<="}: - upper = bound - if op in {"=", "=="}: - equality = value - - if equality is not None: - return None, None, f"{domain_variable} = {equality}" - if lower is None and upper is None: - return None - if lower == float("-inf") and upper is not None: - label = f"< {_format_bucket_bound(upper)}" - elif upper == float("inf") and lower is not None: - label = f">= {_format_bucket_bound(lower)}" - else: - label = f"{_format_bucket_bound(lower)} to {_format_bucket_bound(upper)}" - return lower, upper, label - - -def build_domain_breakdown( - state: AppState, - *, - variable: str | None = None, - domain_variable: str = "adjusted_gross_income", - geo_level: str | None = None, -) -> dict[str, Any]: - targets = _targets_with_inclusion(state) - if targets.empty: - return { - "variable": variable, - "domain_variable": domain_variable, - "geo_level": geo_level, - "rows": [], - "summary": { - "target_count": 0, - "included_target_count": 0, - "evaluated_target_count": 0, - }, - } - - domain_col = targets.get("domain_variable", pd.Series("", index=targets.index)) - mask = domain_col.fillna("").astype(str).str.split(",").apply( - lambda values: domain_variable - in {part.strip() for part in values if part.strip()} - ) - if variable: - mask &= targets.get("variable", pd.Series("", index=targets.index)).astype(str) == variable - if geo_level: - mask &= targets.get("geo_level", pd.Series("", index=targets.index)).astype(str) == geo_level - - scoped = targets[mask].copy() - if scoped.empty: - return { - "variable": variable, - "domain_variable": domain_variable, - "geo_level": geo_level, - "rows": [], - "summary": { - "target_count": 0, - "included_target_count": 0, - "evaluated_target_count": 0, - }, - } - - bucket_values = [] - for _, row in scoped.iterrows(): - bucket = _bucket_from_constraints(_constraints_for_row(row), domain_variable) - if bucket is None: - bucket_values.append((None, None, "Unbucketed")) - else: - bucket_values.append(bucket) - scoped["_bucket_lower"] = [b[0] for b in bucket_values] - scoped["_bucket_upper"] = [b[1] for b in bucket_values] - scoped["_bucket_label"] = [b[2] for b in bucket_values] - - rows = [] - for (lower, upper, label), group in scoped.groupby( - ["_bucket_lower", "_bucket_upper", "_bucket_label"], - dropna=False, - ): - included = group[group.get("included", False).astype(bool)] - summary = _error_summary(included) - rows.append( - { - "bucket": label, - "lower": _finite_float(lower), - "upper": _finite_float(upper), - "target_count": int(len(group)), - "included_target_count": int(len(included)), - "evaluated_target_count": int(summary["evaluated"]), - "median_abs_rel_error": summary["median_abs_rel_error"], - "max_abs_rel_error": summary["max_abs_rel_error"], - "variables": sorted( - group.get("variable", pd.Series("", index=group.index)) - .fillna("") - .astype(str) - .unique() - .tolist() - ), - "geo_levels": sorted( - group.get("geo_level", pd.Series("", index=group.index)) - .fillna("") - .astype(str) - .unique() - .tolist() - ), - } - ) - - rows.sort( - key=lambda row: ( - float("-inf") if row["lower"] is None else row["lower"], - float("inf") if row["upper"] is None else row["upper"], - row["bucket"], - ) - ) - included_all = scoped[scoped.get("included", False).astype(bool)] - summary_all = _error_summary(included_all) - return { - "variable": variable, - "domain_variable": domain_variable, - "geo_level": geo_level, - "rows": rows, - "summary": { - "target_count": int(len(scoped)), - "included_target_count": int(len(included_all)), - "evaluated_target_count": int(summary_all["evaluated"]), - "median_abs_rel_error": summary_all["median_abs_rel_error"], - "max_abs_rel_error": summary_all["max_abs_rel_error"], - }, - } - - -def _node_meta(variable: str, state: AppState) -> dict[str, Any]: - tbs = _tax_benefit_system(state) - tbs_vars = getattr(tbs, "variables", {}) if tbs is not None else {} - var = tbs_vars.get(variable) - targets = _targets_with_inclusion(state) - target_variable = False - domain_variable = False - included_target_count = 0 - target_count = 0 - direct_target_count = 0 - domain_target_count = 0 - evaluated_target_count = 0 - median_abs_rel_error = None - max_abs_rel_error = None - if targets is not None and not targets.empty: - variable_col = targets.get("variable", pd.Series([], dtype=str)).fillna("") - target_mask = variable_col.astype(str) == variable - domain_col = targets.get("domain_variable", pd.Series("", index=targets.index)) - domain_col = domain_col.fillna("").astype(str) - domain_mask = domain_col.str.split(",").apply( - lambda xs: variable in {part.strip() for part in xs if part.strip()} - ) - relevant = targets[target_mask | domain_mask] - included = relevant[relevant.get("included", False).astype(bool)] - summary = _error_summary(included) - target_variable = bool(target_mask.any()) - domain_variable = bool(domain_mask.any()) - direct_target_count = int(target_mask.sum()) - domain_target_count = int(domain_mask.sum()) - target_count = int(len(relevant)) - included_target_count = int(len(included)) - evaluated_target_count = int(summary["evaluated"]) - median_abs_rel_error = summary["median_abs_rel_error"] - max_abs_rel_error = summary["max_abs_rel_error"] - - stored_inputs = set(getattr(_sim_from_state(state), "input_variables", []) or []) - is_formula = bool(getattr(var, "formulas", None)) if var is not None else False - is_aggregate = bool(getattr(var, "adds", None) or getattr(var, "subtracts", None)) - entity = getattr(getattr(var, "entity", None), "key", None) if var is not None else None - label = getattr(var, "label", None) if var is not None else None - - return { - "variable": variable, - "label": label or variable, - "entity": entity, - "is_policyengine_variable": var is not None, - "is_formula": is_formula, - "is_aggregate": is_aggregate, - "is_stored_input": variable in stored_inputs, - "is_target_variable": target_variable, - "is_domain_variable": domain_variable, - "target_count": target_count, - "direct_target_count": direct_target_count, - "domain_target_count": domain_target_count, - "included_target_count": included_target_count, - "evaluated_target_count": evaluated_target_count, - "median_abs_rel_error": median_abs_rel_error, - "max_abs_rel_error": max_abs_rel_error, - } - - -def build_dependency_trace( - variable: str, - state: AppState, - *, - period: int | None = None, - max_nodes: int = 250, -) -> dict[str, Any]: - sim = _sim_from_state(state) - tbs = _tax_benefit_system(state) - if sim is None or tbs is None: - raise RuntimeError("No PolicyEngine simulation is available for this run.") - if variable not in getattr(tbs, "variables", {}): - raise KeyError(variable) - - previous_trace = getattr(sim, "trace", False) - try: - # The PolicyEngine tracer only sees dependencies that are recalculated. - # Cached formula outputs otherwise produce a one-node trace. - invalidate = getattr(sim, "_invalidate_all_caches", None) - if callable(invalidate): - invalidate() - else: - delete_arrays = getattr(sim, "delete_arrays", None) - if callable(delete_arrays): - delete_arrays(variable, period or state.time_period) - sim.trace = True - sim.calculate(variable, period=period or state.time_period) - trace = sim.tracer.get_flat_trace() - finally: - try: - sim.trace = previous_trace - except Exception: - pass - - root_key = next((k for k in trace if _base_trace_key(k) == variable), None) - if root_key is None: - raise RuntimeError(f"Trace did not contain root variable {variable}.") - - reachable: set[str] = set() - ordered: list[str] = [] - depth_by_key = {root_key: 0} - queue = deque([root_key]) - while queue: - key = queue.popleft() - if key in reachable or key not in trace: - continue - reachable.add(key) - ordered.append(key) - for dep in trace[key].get("dependencies", []): - if dep not in depth_by_key: - depth_by_key[dep] = depth_by_key[key] + 1 - queue.append(dep) - - truncated = len(ordered) > max_nodes - ordered = ordered[:max_nodes] - returned = set(ordered) - - nodes = [] - edges = [] - for key in ordered: - variable_name = _base_trace_key(key) - deps = [d for d in trace[key].get("dependencies", []) if d in returned] - meta = _node_meta(variable_name, state) - nodes.append( - { - "id": key, - "variable": variable_name, - "period": _period_from_trace_key(key), - "depth": depth_by_key.get(key, 0), - "dependency_count": len(trace[key].get("dependencies", [])), - "is_leaf": len(trace[key].get("dependencies", [])) == 0, - **meta, - } - ) - edges.extend({"from": key, "to": dep} for dep in deps) - - leaf_nodes = [n for n in nodes if n["is_leaf"]] - summary = { - "total_trace_nodes": int(len(reachable)), - "returned_nodes": int(len(nodes)), - "truncated": truncated, - "leaf_nodes": int(len(leaf_nodes)), - "stored_leaf_nodes": int(sum(n["is_stored_input"] for n in leaf_nodes)), - "targeted_leaf_nodes": int( - sum(n["is_target_variable"] or n["is_domain_variable"] for n in leaf_nodes) - ), - "untargeted_stored_leaf_nodes": int( - sum( - n["is_stored_input"] - and not n["is_target_variable"] - and not n["is_domain_variable"] - for n in leaf_nodes - ) - ), - } - return { - "variable": variable, - "root": root_key, - "summary": summary, - "nodes": nodes, - "edges": edges, - } - - -def audit_target_config( - state: AppState, - *, - variable: str | None = None, -) -> dict[str, Any]: - config, source = _target_config_with_source(state) - config = config or {} - targets = _targets_with_inclusion(state) - rules = list(config.get("include", [])) + list(config.get("exclude", [])) - scope = targets - if variable: - scope = targets[_target_rows_for_variable(targets, variable)].copy() - - out_rules = [] - for section in ("include", "exclude"): - for idx, rule in enumerate(config.get(section, [])): - mask = _target_config_rule_mask(scope, rule) - matched = int(mask.sum()) - if variable and matched == 0 and not _rule_mentions_variable(rule, variable): - continue - out_rules.append( - { - "section": section, - "index": idx, - "rule": rule, - "matched_targets": matched, - "status": "zero_match" if matched == 0 else "matched", - } - ) - - zero = [r for r in out_rules if r["matched_targets"] == 0] - return { - "has_target_config": bool(config), - "target_config_source": source, - "selected_variable": variable, - "target_count": int(len(scope)), - "included_target_count": int( - scope.get("included", pd.Series(False, index=scope.index)) - .fillna(False) - .astype(bool) - .sum() - ), - "rule_count": len(rules), - "zero_match_count": len(zero), - "matched_rule_count": len(out_rules) - len(zero), - "rules": out_rules, - } diff --git a/backend/services/bundle_availability.py b/backend/services/bundle_availability.py deleted file mode 100644 index 5b94b7d..0000000 --- a/backend/services/bundle_availability.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Which calibrated h5 bundles a run actually publishes. - -For staging-layout datasets the pipeline emits an arbitrary subset of -the full bundle catalog (a federal `enhanced_cps_2024.h5`, a -`national/US.h5`, 51 `states/.h5`, 436 `districts/.h5`). -GHA-only runs typically publish just the federal one; tagged releases -publish all 499. - -Listing the published set lets the dashboard: - -1. Show only bundles that exist when the user filters /targets by - dataset, so we stop labeling rows with files that aren't there. -2. Decide whether per-bundle Microsim evaluation is even possible for - the loaded run, and short-circuit cleanly when it isn't. - -Results are cached per (repo_id, run_id) since the published artifact -set is immutable for a given run. -""" - -from __future__ import annotations - -import logging -from typing import Iterable - -logger = logging.getLogger(__name__) - -# (repo_id, run_id) -> set of bundle paths, e.g. {"enhanced_cps_2024.h5", -# "states/CA.h5", "districts/CA-12.h5"} -_CACHE: dict[tuple[str, str], frozenset[str]] = {} - - -def _bundle_paths_for_run(repo_id: str, run_id: str) -> frozenset[str]: - """Hit the HF repo file listing and pick out h5 paths under the run.""" - from huggingface_hub import HfApi - api = HfApi() - try: - files = api.list_repo_files(repo_id, repo_type="model") - except Exception as exc: - logger.warning("HF list_repo_files failed for %s: %s", repo_id, exc) - return frozenset() - root_run = run_id == "main" - current_staging = run_id == "staging" - if root_run: - prefix = "" - elif current_staging: - prefix = "staging/" - else: - prefix = f"staging/{run_id}/" - bundles: set[str] = set() - for f in files: - if not f.startswith(prefix) or not f.endswith(".h5"): - continue - rel = f[len(prefix):] - if current_staging: - if rel == "calibration/source_imputed_stratified_extended_cps.h5": - bundles.add("source_imputed_stratified_extended_cps.h5") - continue - if "/" not in rel or rel.split("/", 1)[0] not in { - "cities", "districts", "national", "states", - }: - continue - if root_run and "/" in rel and rel.split("/", 1)[0] not in { - "cities", "districts", "national", "states", - }: - continue - # Skip clone-diagnostics-style sidecars that share the .h5 stem. - if rel.endswith(".clone_diagnostics.json"): - continue - bundles.add(rel) - return frozenset(bundles) - - -def published_bundles(repo_id: str, run_id: str) -> frozenset[str]: - """Cached: return the set of bundle paths under a run.""" - key = (repo_id, run_id) - cached = _CACHE.get(key) - if cached is not None: - return cached - bundles = _bundle_paths_for_run(repo_id, run_id) - _CACHE[key] = bundles - return bundles - - -def filter_to_available( - candidates: Iterable[str], - repo_id: str, - run_id: str, -) -> list[str]: - """Drop bundle names from `candidates` that the run doesn't publish. - - Used to scope the dataset-file facet on /targets/facets to what's - actually fetchable for this run. - """ - available = published_bundles(repo_id, run_id) - if not available: - return list(candidates) - return [c for c in candidates if c in available] diff --git a/backend/services/bundle_eval.py b/backend/services/bundle_eval.py deleted file mode 100644 index 8971c82..0000000 --- a/backend/services/bundle_eval.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Per-bundle PE-aggregate evaluation. - -The dataset_loader builds one Microsim from the federal -``enhanced_cps_2024.h5``. The per-state and per-district bundles -published by versioned us-data runs hold *different* calibrated -weights — so the federal-fit PE aggregate is **not** the per-bundle -PE aggregate. To validate "is this dataset well-calibrated against its -targets", we have to re-evaluate against the bundle's own h5. - -This module: - -- Lazily downloads each bundle h5 from HF (cache to disk). -- Loads each into its own ``Microsimulation`` instance (LRU cached in - memory, max 10). -- Runs the entity-aware evaluator over the subset of targets that - belong to the bundle, returning per-row ``estimate`` / - ``rel_error`` / ``abs_rel_error`` evaluated against THAT h5. -- Persists results to a per-bundle pickle so subsequent picks of the - same bundle skip both the h5 load and the evaluator pass. - -Triggered from ``/targets`` only when the caller filters to exactly -one bundle that actually exists for the run; multi-bundle and -no-filter calls keep the federal-fit numbers. -""" - -from __future__ import annotations - -import logging -import shutil -import threading -from collections import OrderedDict -from pathlib import Path - -import pandas as pd - -logger = logging.getLogger(__name__) - -# LRU of bundle Microsims, keyed by (repo_id, run_id, bundle_path). -# Bounded so we don't exhaust memory if a user clicks through many -# states / districts in a session. -_SIM_CACHE: "OrderedDict[tuple[str, str, str], object]" = OrderedDict() -_SIM_LOCK = threading.Lock() -_SIM_MAX = 10 - - -def _h5_local_path( - repo_id: str, - run_id: str, - bundle: str, - cache_root: str, -) -> Path: - """Lazily fetch the bundle's h5 from HF and return its local path.""" - repo_slug = repo_id.replace("/", "__") - cache = ( - Path(cache_root) / repo_slug / "root" / run_id - if run_id == "main" - else Path(cache_root) / repo_slug / "staging" / run_id - ) - cache.mkdir(parents=True, exist_ok=True) - local = cache / bundle # nested in cache, e.g. states/CA.h5 - if local.exists(): - return local - - from huggingface_hub import hf_hub_download - if run_id == "main": - hf_path = bundle - elif run_id == "staging": - hf_path = f"staging/{bundle}" - else: - hf_path = f"staging/{run_id}/{bundle}" - logger.info("Downloading bundle h5 %s/%s", repo_id, hf_path) - downloaded = hf_hub_download( - repo_id=repo_id, - filename=hf_path, - repo_type="model", - local_dir=str(cache), - ) - src = Path(downloaded) - if src != local: - local.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(src), str(local)) - return local - - -def _get_sim( - repo_id: str, - run_id: str, - bundle: str, - cache_root: str = ".artifacts", -): - """LRU-cached Microsim loaded from the bundle's h5.""" - key = (repo_id, run_id, bundle) - with _SIM_LOCK: - cached = _SIM_CACHE.get(key) - if cached is not None: - _SIM_CACHE.move_to_end(key) - return cached - - h5_path = _h5_local_path(repo_id, run_id, bundle, cache_root) - from policyengine_us import Microsimulation - logger.info("Initialising bundle Microsim from %s", h5_path) - sim = Microsimulation(dataset=str(h5_path)) - - with _SIM_LOCK: - _SIM_CACHE[key] = sim - _SIM_CACHE.move_to_end(key) - while len(_SIM_CACHE) > _SIM_MAX: - evicted_key, _ = _SIM_CACHE.popitem(last=False) - logger.info("Evicted bundle sim %s from LRU", evicted_key) - return sim - - -def _estimate_cache_path( - repo_id: str, - run_id: str, - bundle: str, - cache_root: str = ".artifacts", -) -> Path: - repo_slug = repo_id.replace("/", "__") - cache = ( - Path(cache_root) / repo_slug / "root" / run_id - if run_id == "main" - else Path(cache_root) / repo_slug / "staging" / run_id - ) - cache.mkdir(parents=True, exist_ok=True) - safe = bundle.replace("/", "__") - return cache / f"{safe}.bundle_estimates.pkl" - - -def evaluate_bundle( - targets_df: pd.DataFrame, - *, - repo_id: str, - run_id: str, - bundle: str, - time_period: int = 2024, - cache_root: str = ".artifacts", -) -> pd.DataFrame: - """Re-evaluate ``targets_df`` against ``bundle``'s h5. - - Mutates a copy and returns it with the federal ``estimate`` / - ``rel_error`` / ``abs_rel_error`` columns overridden by per-bundle - numbers. Uses a per-bundle pickle cache so the second pick of the - same bundle is effectively instant. - """ - out = targets_df.copy() - cache_path = _estimate_cache_path(repo_id, run_id, bundle, cache_root) - - # Fast path: cached estimates. - if cache_path.exists(): - try: - cached = pd.read_pickle(cache_path) - joined = out.drop( - columns=["estimate", "rel_error", "abs_rel_error"], - errors="ignore", - ).merge( - cached[["target_id", "estimate", "rel_error", "abs_rel_error"]], - on="target_id", - how="left", - ) - logger.info( - "Loaded cached bundle estimates (%d rows) for %s", - len(cached), bundle, - ) - return joined - except Exception as exc: - logger.warning( - "Bundle estimate cache read failed (%s); recomputing.", exc, - ) - - sim = _get_sim(repo_id, run_id, bundle, cache_root) - from backend.services.stratum_evaluator import evaluate_targets - evaluated = evaluate_targets(out, sim, default_period=time_period) - - slim = evaluated[ - ["target_id", "estimate", "rel_error", "abs_rel_error"] - ].copy() - try: - slim.to_pickle(cache_path) - logger.info( - "Cached bundle %s estimates (%d rows) → %s", - bundle, len(slim), cache_path, - ) - except Exception as exc: - logger.warning("Failed to cache bundle estimates: %s", exc) - return evaluated diff --git a/backend/services/dataset_loader.py b/backend/services/dataset_loader.py deleted file mode 100644 index ad0df17..0000000 --- a/backend/services/dataset_loader.py +++ /dev/null @@ -1,825 +0,0 @@ -"""Load a calibration run from the canonical us-data staging layout. - -Unlike loader.py (which depends on calibration_package.pkl), this: -- downloads policy_data.db (targets DB) and one h5 dataset per run -- reads targets directly from the DB -- defers estimate computation to the stratum evaluator (Step 3) - -The resulting AppState has X_csr / X_csc / initial_weights left empty; routes -that need the X matrix (per-target detail tabs) won't function for these runs, -but the universe view (Summary + All targets) works. -""" - -from __future__ import annotations - -import logging -import os -import re -from pathlib import Path -from typing import TYPE_CHECKING - -import numpy as np -import pandas as pd -from sqlalchemy import create_engine - -from backend.services.runs import DatasetConfig -from backend.state import AppState - -if TYPE_CHECKING: - from huggingface_hub import HfApi # noqa: F401 - -logger = logging.getLogger(__name__) - - -_CONSTRAINT_RE = re.compile(r"([A-Za-z_][\w]*)(==|!=|>=|<=|>|<)(.+)$") - - -def _parse_constraint(s: str) -> tuple[str, str, str] | None: - """Parse 'var op value' (with or without spaces) into a tuple.""" - s = s.strip().replace(" ", "") - m = _CONSTRAINT_RE.match(s) - if not m: - return None - var, op, val = m.group(1), m.group(2), m.group(3).strip() - return (var, op, _norm_constraint_value(val)) - - -def _norm_constraint_value(v: str) -> str: - """Normalise numeric constraint values: '1.0' → '1', '-inf' stays '-inf'.""" - s = str(v).strip() - if s in ("inf", "-inf", "Infinity", "-Infinity"): - return s.replace("Infinity", "inf") - try: - f = float(s) - if f != f or f in (float("inf"), float("-inf")): - return s - if f == int(f): - return str(int(f)) - return str(f) - except (TypeError, ValueError): - return s - - -def _parse_diagnostics_key(key: str) -> tuple | None: - """Convert a diagnostics CSV target key into an order-independent signature. - - Examples: - national/medicaid → ('national', '', 'medicaid', ()) - national/adj.../[tax_unit_is_filer==1] → ('national', '', 'adj...', (('tax_unit_is_filer','==','1'),)) - cd_1000/person_count/[age<5,age>-1] → ('district', '1000', 'person_count', (('age','<','5'),('age','>','-1'))) - """ - if not isinstance(key, str) or "/" not in key: - return None - head, _, rest = key.partition("/") - if head == "national": - geo_level, geo_id = "national", "" - elif head.startswith("cd_"): - geo_level, geo_id = "district", head[3:] - else: - geo_level, geo_id = head, "" - if "/" in rest: - variable, _, constraint_part = rest.partition("/") - else: - variable, constraint_part = rest, "" - constraints: list[tuple[str, str, str]] = [] - if constraint_part.startswith("[") and constraint_part.endswith("]"): - body = constraint_part[1:-1].strip() - if body: - for c in body.split(","): - parsed = _parse_constraint(c) - if parsed is not None: - constraints.append(parsed) - return (geo_level, geo_id, variable, tuple(sorted(constraints))) - - -def _row_to_diagnostics_signature(row) -> tuple: - """Same signature shape as _parse_diagnostics_key but built from a - targets_enriched row. Order-independent.""" - variable = str(row.get("variable") or "") - geo_level = row.get("geo_level") or "national" - gid = row.get("geographic_id") - if gid is None or (isinstance(gid, float) and pd.isna(gid)): - geo_id_norm = "" - else: - try: - geo_id_norm = str(int(float(str(gid)))) - except (TypeError, ValueError): - geo_id_norm = str(gid) - cons = row.get("constraints") or [] - parsed_cons = [] - for c in cons: - if isinstance(c, (list, tuple)) and len(c) >= 3: - parsed_cons.append((str(c[0]), str(c[1]), - _norm_constraint_value(str(c[2])))) - elif isinstance(c, str): - p = _parse_constraint(c) - if p is not None: - parsed_cons.append(p) - return (geo_level, geo_id_norm, variable, tuple(sorted(parsed_cons))) - - -def _target_to_diagnostics_key(row) -> str: - """Build the target key used in unified_diagnostics.csv from a - targets_enriched row. - - Diag format observed empirically: - / (no constraints) - //[,,...] (with constraints, no spaces) - - Where geo_prefix is: - "national" — when no geographic constraint - "cd_" — when geographic_id is a district id (no leading zeros) - - State-level rows aren't represented in the diagnostics file (the - calibration trains national + district only). We return a key anyway; - those will simply miss the join. - """ - variable = str(row.get("variable") or "") - geo_level = row.get("geo_level") or "" - gid = row.get("geographic_id") - if geo_level == "national" or geo_level == "" or gid is None or (isinstance(gid, float) and pd.isna(gid)): - prefix = "national" - elif geo_level == "district": - try: - prefix = f"cd_{int(float(str(gid)))}" - except (TypeError, ValueError): - prefix = f"cd_{gid}" - elif geo_level == "state": - # No state rows in diag — return a key that won't match. - try: - prefix = f"state_{int(float(str(gid)))}" - except (TypeError, ValueError): - prefix = f"state_{gid}" - else: - prefix = geo_level - - cons = row.get("constraints") or [] - if not cons: - return f"{prefix}/{variable}" - # Strip spaces from each "var op value" → "varopvalue"; join with ",". - parts = [] - for c in cons: - # `c` may be either a string "var op value" (from DB rebuild) or a - # (var, op, value) tuple. - if isinstance(c, (list, tuple)) and len(c) >= 3: - parts.append(f"{c[0]}{c[1]}{c[2]}") - else: - parts.append(str(c).replace(" ", "")) - return f"{prefix}/{variable}/[{','.join(parts)}]" - - -def _try_fetch_unified_diagnostics( - dataset, - run_id: str, - cache_root: str = ".artifacts", -) -> "tuple[pd.DataFrame, str] | None": - """For staging-layout datasets, try to fetch the canonical post- - calibration diagnostics CSV from the repo. - - Uses the Stage 3 artifact catalog (mirrored locally as - ``backend.services.fit_artifacts`` until us-data ships a release that - exports ``policyengine_us_data.fit_weights.artifacts``). Tries both - regional (`unified_diagnostics.csv`) and national - (`national_unified_diagnostics.csv`) filenames so we don't silently - miss national-scope runs. - - For each scope we look in: - 1. `calibration/runs//diagnostics/` — per-run. - 2. `calibration/logs/` — "current" canonical snapshot, but only - for the latest discovered run. Reusing current logs for older staging - runs silently reports the wrong calibration fit. - - Returns ``(df, scope)`` on success — knowing the scope lets later code - pick the matching weights / run_config files. Returns ``None`` if - nothing was found. - """ - if dataset.layout == "root": - from huggingface_hub import hf_hub_download - from backend.services.fit_artifacts import artifacts_for_scope - - filename = artifacts_for_scope("regional").diagnostics - path = f"calibration/logs/{filename}" - repo_slug = dataset.repo_id.replace("/", "__") - cache = Path(cache_root) / repo_slug - cache.mkdir(parents=True, exist_ok=True) - try: - local = hf_hub_download( - repo_id=dataset.repo_id, - filename=path, - repo_type=dataset.repo_type, - local_dir=str(cache), - ) - df = pd.read_csv(local) - if "target" in df.columns and "estimate" in df.columns: - logger.info( - "Loaded root production diagnostics from %s (%d rows)", - path, len(df), - ) - return df, "regional" - except Exception as exc: - logger.debug("diagnostics fetch failed at %s: %s", path, exc) - return None - - if dataset.layout != "staging": - return None - from huggingface_hub import hf_hub_download - from backend.services.fit_artifacts import ( - artifacts_for_scope, - SCOPES, - ) - - repo_slug = dataset.repo_id.replace("/", "__") - cache = Path(cache_root) / repo_slug - cache.mkdir(parents=True, exist_ok=True) - - for scope in SCOPES: - filename = artifacts_for_scope(scope).diagnostics - candidate_paths = [ - f"calibration/runs/{run_id}/diagnostics/{filename}", - ] - try: - from backend.services import runs as runs_service - latest = next(iter(runs_service.list_runs(dataset.id)), None) - if latest is not None and latest.run_id == run_id: - candidate_paths.append(f"calibration/logs/{filename}") - except Exception as exc: - logger.debug("Could not determine latest run for diagnostics fallback: %s", exc) - for path in candidate_paths: - try: - local = hf_hub_download( - repo_id=dataset.repo_id, - filename=path, - repo_type=dataset.repo_type, - local_dir=str(cache), - ) - df = pd.read_csv(local) - if "target" in df.columns and "estimate" in df.columns: - logger.info( - "Loaded %s-scope diagnostics from %s (%d rows)", - scope, path, len(df), - ) - return df, scope - except Exception as exc: - logger.debug("diagnostics fetch failed at %s: %s", path, exc) - continue - return None - - -def _join_diagnostics( - targets_df: pd.DataFrame, - diag_df: pd.DataFrame, -) -> int: - """Attach published calibration diagnostics to DB targets. - - Diagnostics keys do not carry target_id or period, and the DB can contain - multiple active rows with the same variable/geography/constraint signature. - Join one diagnostics row to at most one DB row, picking the closest target - value when duplicates exist. For joined rows, the diagnostics true_value is - the calibrated target value, so use that value for package-derived error - calculations instead of the raw DB source value. - """ - sig_to_indices: dict[tuple, list[int]] = {} - for idx, row in targets_df.iterrows(): - sig_to_indices.setdefault(_row_to_diagnostics_signature(row), []).append(idx) - - matched_indices: set[int] = set() - n_joined = 0 - has_true = "true_value" in diag_df.columns - has_rel = "rel_error" in diag_df.columns - has_abs = "abs_rel_error" in diag_df.columns - - for _, diag_row in diag_df.iterrows(): - sig = _parse_diagnostics_key(diag_row.get("target")) - if sig is None: - continue - candidates = [ - idx for idx in sig_to_indices.get(sig, []) - if idx not in matched_indices - ] - if not candidates: - continue - - true_value = diag_row.get("true_value") if has_true else np.nan - if pd.notna(true_value): - def distance(idx: int) -> float: - raw_value = targets_df.at[idx, "value"] - try: - return abs(float(raw_value) - float(true_value)) / max( - 1.0, abs(float(raw_value)) - ) - except (TypeError, ValueError): - return float("inf") - chosen = min(candidates, key=distance) - else: - chosen = candidates[0] - - matched_indices.add(chosen) - if pd.notna(true_value): - targets_df.at[chosen, "value"] = float(true_value) - targets_df.at[chosen, "estimate"] = diag_row.get("estimate") - if has_rel and pd.notna(diag_row.get("rel_error")): - targets_df.at[chosen, "rel_error"] = float(diag_row["rel_error"]) - if has_abs and pd.notna(diag_row.get("abs_rel_error")): - targets_df.at[chosen, "abs_rel_error"] = float(diag_row["abs_rel_error"]) - targets_df.at[chosen, "included"] = True - if has_rel and pd.notna(diag_row.get("rel_error")): - targets_df.at[chosen, "loss_contribution"] = float(diag_row["rel_error"]) ** 2 - n_joined += 1 - - return n_joined - - -def _household_series(sim, variable: str, period: int, default=None) -> np.ndarray: - try: - return np.asarray( - sim.calculate(variable, map_to="household", period=period).values - ) - except Exception: - if default is None: - raise - return np.asarray(default) - - -def _ensure_staging_artifacts( - dataset: DatasetConfig, - run_id: str, - cache_root: str = ".artifacts", -) -> dict[str, str]: - """Download a staging run's required files; return a {logical_name: path} map. - - Logical names are the flat filenames (``policy_data.db``, - ``enhanced_cps_2024.h5``); on HF they may live at the run root OR - nested under ``calibration/`` / ``datasets/``. We probe both so - versioned releases and GHA staging both load through the same code - path, and store everything flat in the local cache. - """ - from huggingface_hub import hf_hub_download - from backend.services.runs import ( - _resolve_staging_file_paths, - _resolve_staging_root_file_paths, - ) - import shutil - - repo_slug = dataset.repo_id.replace("/", "__") - if dataset.layout == "root": - cache = Path(cache_root) / repo_slug / "root" / run_id - else: - cache = Path(cache_root) / repo_slug / "staging" / run_id - cache.mkdir(parents=True, exist_ok=True) - - logical_names = list(dataset.effective_required_files()) - if dataset.layout == "root": - actual_paths = {name: name for name in logical_names} - elif dataset.layout == "staging-root": - actual_paths = _resolve_staging_root_file_paths( - dataset.repo_id, - logical_names, - ) - else: - actual_paths = _resolve_staging_file_paths( - dataset.repo_id, - run_id, - logical_names, - ) - - resolved: dict[str, str] = {} - for fn in logical_names: - dest = cache / fn # flat in our cache - if dest.exists(): - logger.info("Found cached %s: %s", fn, dest) - resolved[fn] = str(dest) - continue - - hf_path = actual_paths.get(fn) - if hf_path is None: - logger.warning("Run %s does not publish %s; skipping.", run_id, fn) - continue - logger.info("Downloading %s from %s/%s ...", fn, dataset.repo_id, hf_path) - try: - downloaded = hf_hub_download( - repo_id=dataset.repo_id, - filename=hf_path, - repo_type=dataset.repo_type, - local_dir=str(cache), - ) - except Exception as exc: - logger.warning("Could not download %s: %s", hf_path, exc) - continue - - # hf_hub_download honors the repo path under local_dir, so the - # file lands at cache/. Move it to the flat dest so the - # rest of the loader doesn't need to know about the layout. - src = Path(downloaded) - if src != dest: - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(src), str(dest)) - if dest.exists(): - resolved[fn] = str(dest) - size_mb = dest.stat().st_size / 1e6 - logger.info(" Downloaded %s (%.1f MB)", fn, size_mb) - - return resolved - - -def _load_targets_from_db(db_engine) -> tuple[pd.DataFrame, list[str]]: - """Read targets + denormalised constraint info from policy_data.db. - - Returns (targets_df, target_names) where targets_df mirrors the columns - the rest of the dashboard expects: target_id, variable, value, period, - geo_level, geographic_id, domain_variable, included, plus a 'constraints' - list and a 'target_name' string built from those. - """ - # All active targets (DB convention — calibration team uses `active` flag) - targets = pd.read_sql( - "SELECT target_id, variable, period, stratum_id, value, active, " - "tolerance, source, notes FROM targets WHERE active = 1", - db_engine, - ) - - # Pull all constraints once, group by stratum_id (faster than per-row query). - constraints = pd.read_sql( - "SELECT stratum_id, constraint_variable, operation, value FROM " - "stratum_constraints", - db_engine, - ) - - geo_vars = {"state_fips", "congressional_district_geoid", "ucgid_str"} - by_stratum: dict[int, list[dict]] = {} - for _, row in constraints.iterrows(): - by_stratum.setdefault(row.stratum_id, []).append({ - "variable": row.constraint_variable, - "operation": row.operation, - "value": row.value, - }) - - geo_levels: list[str] = [] - geographic_ids: list[str | None] = [] - domain_vars: list[str | None] = [] - constraint_lists: list[list[str]] = [] - - for sid in targets["stratum_id"]: - cons = by_stratum.get(int(sid), []) - geo_con = next((c for c in cons if c["variable"] in geo_vars), None) - if geo_con is None: - geo_level = "national" - geographic_id = None - elif geo_con["variable"] == "state_fips": - geo_level = "state" - geographic_id = str(geo_con["value"]) - elif geo_con["variable"] == "congressional_district_geoid": - geo_level = "district" - geographic_id = str(geo_con["value"]) - else: - geo_level = geo_con["variable"] - geographic_id = str(geo_con["value"]) - - non_geo = [c for c in cons if c["variable"] not in geo_vars] - domain_var = ",".join(sorted({c["variable"] for c in non_geo})) or None - readable = [f"{c['variable']} {c['operation']} {c['value']}" for c in non_geo] - - geo_levels.append(geo_level) - geographic_ids.append(geographic_id) - domain_vars.append(domain_var) - constraint_lists.append(readable) - - targets["geo_level"] = geo_levels - targets["geographic_id"] = geographic_ids - targets["domain_variable"] = domain_vars - targets["constraints"] = constraint_lists - - # Build readable target names: ///[constraints] - names: list[str] = [] - for _, r in targets.iterrows(): - gid = r["geographic_id"] - if gid is None or (isinstance(gid, float) and pd.isna(gid)): - geo_part = "US" - else: - geo_part = str(gid) - constraint_part = ( - ",".join(r["constraints"]) if r["constraints"] else "" - ) - names.append( - f"{r['geo_level']}/{r['variable']}/{geo_part}/[{constraint_part}]" - ) - targets["target_name"] = names - - # Convenience aliased columns matching the pkl-mode shape - targets["target_idx"] = np.arange(len(targets)) - # `included` is set to True later only for rows that match a published - # entry in unified_diagnostics.csv — the only honest signal we have for - # "this target was actually evaluated by the calibration loss this run." - # The DB's `active` flag is always 1 and so isn't useful here. - targets["included"] = False - targets["estimate"] = np.nan - targets["rel_error"] = np.nan - targets["abs_rel_error"] = np.nan - targets["loss_contribution"] = 0.0 - targets["n_contributors"] = 0 - - return targets, names - - -def _detect_time_period(sim) -> int: - try: - raw_keys = sim.dataset.load_dataset()["household_id"] - if isinstance(raw_keys, dict): - return int(next(iter(raw_keys))) - except Exception: - pass - return 2024 - - - - -def load_run_from_dataset( - dataset: DatasetConfig, - run_id: str, - cache_root: str = ".artifacts", -) -> AppState: - """Load a staging-layout run into an AppState. Step 2 of the refactor: - populates targets + DB + simulation but leaves estimates as NaN (Step 4 - will fill them in via the stratum evaluator). - """ - from policyengine_us import Microsimulation - - files = _ensure_staging_artifacts(dataset, run_id, cache_root) - if "policy_data.db" not in files or dataset.primary_h5 not in files: - raise RuntimeError( - f"Required files missing for {dataset.id}/{run_id}: " - f"have {sorted(files)}" - ) - - logger.info("Connecting to policy_data.db at %s", files["policy_data.db"]) - db_engine = create_engine(f"sqlite:///{files['policy_data.db']}") - - logger.info("Loading targets from DB...") - targets_df, target_names = _load_targets_from_db(db_engine) - logger.info("Loaded %d active targets", len(targets_df)) - - logger.info("Initializing Microsimulation from %s", files[dataset.primary_h5]) - sim = Microsimulation(dataset=files[dataset.primary_h5]) - time_period = _detect_time_period(sim) - - # If we've previously computed estimates for this run, the parquet - # cache is good for the lifetime of the run + selected h5 (run_id is - # immutable per publish). Skip the CSV join + entity-aware evaluator - # entirely. - # Pickle keeps pandas types (object cols with lists, etc.) and needs no - # extra deps. Cache key is the immutable run_id, so no staleness risk - # within a run; bump SCHEMA_VERSION below to invalidate on shape change. - cache_stem = Path(dataset.primary_h5).stem.replace("/", "_") - layout_cache_dir = "root" if dataset.layout == "root" else "staging" - enriched_cache_path = ( - Path(cache_root) / dataset.repo_id.replace("/", "__") - / layout_cache_dir / run_id / f"targets_enriched.{cache_stem}.pkl" - ) - SCHEMA_VERSION = 4 - cached_enriched: pd.DataFrame | None = None - detected_scope_cached: str | None = None - if enriched_cache_path.exists(): - try: - meta_path = enriched_cache_path.with_suffix(".meta.json") - if meta_path.exists(): - import json as _json - meta = _json.loads(meta_path.read_text()) - if meta.get("version") == SCHEMA_VERSION: - cached_enriched = pd.read_pickle(enriched_cache_path) - detected_scope_cached = meta.get("fit_scope") - logger.info( - "Loaded cached targets_enriched (%d rows) from %s", - len(cached_enriched), enriched_cache_path, - ) - except Exception as exc: - logger.warning("Failed to read enriched cache (%s); recomputing.", exc) - cached_enriched = None - - # Household-level scaffolding so weights/geo lookups work - household_weight = sim.calculate( - "household_weight", map_to="household", period=time_period, - ).values - n_households = len(household_weight) - - compute_household_fields = os.environ.get( - "COMPUTE_HOUSEHOLD_FIELDS", "", - ).lower() in {"1", "true", "yes"} - if compute_household_fields: - state_fips = _household_series( - sim, "state_fips", time_period, default=np.zeros(n_households) - ).astype(int) - cd_geoid = _household_series( - sim, "congressional_district_geoid", time_period, - default=np.zeros(n_households), - ).astype(int) - hh_income = _household_series( - sim, "spm_unit_net_income", time_period, default=np.zeros(n_households) - ) - hh_threshold = _household_series( - sim, "spm_unit_spm_threshold", time_period, default=np.zeros(n_households) - ) - in_poverty = hh_income < hh_threshold - try: - income_decile = pd.qcut(hh_income, 10, labels=False, duplicates="drop") - income_decile = np.asarray(income_decile).astype(np.int8) - except Exception: - income_decile = np.zeros(n_households, dtype=np.int8) - else: - logger.info( - "Skipping household income/geography fields " - "(set COMPUTE_HOUSEHOLD_FIELDS=true to compute them)." - ) - state_fips = np.zeros(n_households, dtype=int) - cd_geoid = np.zeros(n_households, dtype=int) - hh_income = np.zeros(n_households, dtype=np.float32) - hh_threshold = np.zeros(n_households, dtype=np.float32) - in_poverty = np.zeros(n_households, dtype=bool) - income_decile = np.zeros(n_households, dtype=np.int8) - - households_df = pd.DataFrame({ - "household_idx": np.arange(n_households), - "income": hh_income.astype(np.float32), - "spm_threshold": hh_threshold.astype(np.float32), - "in_poverty": in_poverty, - "initial_weight": household_weight.astype(np.float32), - "final_weight": household_weight.astype(np.float32), - "g_weight": np.ones(n_households, dtype=np.float32), - "state": state_fips, - "cd_geoid": cd_geoid, - "income_decile": income_decile, - }) - - # Dataset mode: the canonical us-data repo publishes a per-run - # `unified_diagnostics.csv` (under calibration/runs//diagnostics/). - # We use it to identify targets that were actually in the calibration loss - # and to pick the published true_value, then recompute PE aggregates from - # the h5 with policyengine_us so the dashboard's "PE aggregate" column is - # sourced from the same package API users would call manually. - # - # Targets the diagnostics file doesn't cover (typically excluded by - # target_config.yaml during this calibration) are left unestimated unless - # explicitly requested; evaluating all authored targets is slow and - # misleading for "included targets" diagnostics. - # Skip CSV join + evaluator entirely if the parquet cache was already - # warmed for this run. targets_df becomes the cached frame; we still - # need to compute downstream stuff (households_df, sparse mats). - if cached_enriched is not None: - targets_df = cached_enriched - diag_result = None - diag_scope = detected_scope_cached - else: - diag_result = _try_fetch_unified_diagnostics(dataset, run_id, cache_root) - diag_scope: str | None = None - if diag_result is not None: - diag_df, diag_scope = diag_result - logger.info( - "Joining %d rows from %s-scope diagnostics onto %d targets...", - len(diag_df), diag_scope, len(targets_df), - ) - # A CSV match means the pipeline evaluated this target in its loss. - # Join one-to-one: target_id is absent from diagnostics and signatures - # can repeat across source periods in policy_data.db. - n_from_diag = _join_diagnostics(targets_df, diag_df) - logger.info( - " → %d/%d targets got estimates from diagnostics CSV (marked included=True)", - n_from_diag, len(targets_df), - ) - elif cached_enriched is None: - logger.info("No unified_diagnostics.csv found; falling back to MVP evaluator only.") - - compute_pe_aggregates = os.environ.get( - "COMPUTE_PE_AGGREGATES", "true", - ).lower() not in {"0", "false", "no"} - if ( - cached_enriched is None - and diag_result is not None - and compute_pe_aggregates - and targets_df["included"].any() - ): - from backend.services.stratum_evaluator import evaluate_targets - - included_mask = targets_df["included"].astype(bool) - logger.info( - "Computing PE aggregates from published h5 for %d in-loss targets...", - int(included_mask.sum()), - ) - evaluated = evaluate_targets( - targets_df[included_mask].copy(), - sim, - default_period=time_period, - ) - idx = evaluated.index - targets_df.loc[idx, "estimate"] = evaluated["estimate"].values - targets_df.loc[idx, "rel_error"] = evaluated["rel_error"].values - targets_df.loc[idx, "abs_rel_error"] = evaluated["abs_rel_error"].values - if "eval_note" in evaluated.columns: - targets_df.loc[idx, "eval_note"] = evaluated["eval_note"].values - with np.errstate(invalid="ignore"): - targets_df.loc[idx, "loss_contribution"] = ( - targets_df.loc[idx, "rel_error"].astype(float) ** 2 - ) - logger.info( - "Computed PE aggregates for %d/%d in-loss targets.", - int(targets_df.loc[idx, "estimate"].notna().sum()), - int(included_mask.sum()), - ) - - # Fill remaining NaN estimates only when diagnostics were unavailable, or - # explicitly requested. Evaluating tens of thousands of skipped/authored - # targets through Microsimulation makes normal dashboard loads look hung. - eval_skipped = os.environ.get("EVALUATE_SKIPPED_TARGETS", "").lower() in { - "1", "true", "yes", - } - should_eval_remaining = cached_enriched is None and ( - (diag_result is None and dataset.layout != "staging-root") or eval_skipped - ) - if targets_df["estimate"].isna().any() and should_eval_remaining: - from backend.services.stratum_evaluator import evaluate_targets - unfilled = targets_df["estimate"].isna() - logger.info( - "Running MVP evaluator on %d remaining targets...", int(unfilled.sum()), - ) - filled = evaluate_targets( - targets_df[unfilled].copy(), sim, default_period=time_period, - ) - targets_df.loc[unfilled, "estimate"] = filled["estimate"].values - elif targets_df["estimate"].isna().any(): - logger.info( - "Skipping MVP evaluator for %d non-diagnostics targets " - "(set EVALUATE_SKIPPED_TARGETS=true to compute them).", - int(targets_df["estimate"].isna().sum()), - ) - n_evaluated = int(np.sum(~targets_df["estimate"].isna())) - logger.info( - "Total estimates available: %d/%d (%.1f%%)", - n_evaluated, len(targets_df), 100 * n_evaluated / max(1, len(targets_df)), - ) - - # Compute rel_error / abs_rel_error for rows not populated by diagnostics. - target_values = targets_df["value"].to_numpy(dtype=np.float64) - estimates_arr = targets_df["estimate"].to_numpy(dtype=np.float64) - with np.errstate(divide="ignore", invalid="ignore"): - rel = np.where( - np.abs(target_values) > 0, - (estimates_arr - target_values) / np.abs(target_values), - np.nan, - ) - targets_df["rel_error"] = np.where( - targets_df["rel_error"].notna(), targets_df["rel_error"], rel, - ) - targets_df["abs_rel_error"] = np.where( - targets_df["abs_rel_error"].notna(), - targets_df["abs_rel_error"], - np.abs(rel), - ) - - # Stash the detected fit scope so other artifact lookups can pick the - # right regional/national filenames. Falls back to regional when we - # couldn't fetch diagnostics at all (best guess for legacy runs). - detected_scope = diag_scope or "regional" - - # Write the enriched parquet cache so subsequent loads can skip the - # CSV join + evaluator entirely. Only write if we actually did the - # work (cached_enriched is None means we recomputed this load). - if cached_enriched is None: - try: - enriched_cache_path.parent.mkdir(parents=True, exist_ok=True) - targets_df.to_pickle(enriched_cache_path) - import json as _json - enriched_cache_path.with_suffix(".meta.json").write_text( - _json.dumps({"fit_scope": detected_scope, "version": SCHEMA_VERSION}) - ) - logger.info("Cached targets_enriched → %s", enriched_cache_path) - except Exception as exc: - logger.warning("Failed to cache enriched targets (%s); continuing.", exc) - - # Empty sparse matrices since dataset mode doesn't have the pipeline's - # X matrix yet (waiting on data-team publish). - from scipy import sparse - X_csr = sparse.csr_matrix((len(targets_df), n_households)) - X_csc = X_csr.tocsc() - - state = AppState( - X_csr=X_csr, - X_csc=X_csc, - targets_df=targets_df, - target_names=target_names, - targets_enriched=targets_df, - households_df=households_df, - sim_service=None, # filled in step 3 alongside the evaluator - db_engine=db_engine, - time_period=time_period, - n_targets=len(targets_df), - n_households=n_households, - dataset_id=dataset.id, - run_id=run_id, - # Weights live on the dataset itself; we don't have a "before - # calibration" view because the published dataset IS the post- - # calibration state. - initial_weights=household_weight, - final_weights=household_weight, - g_weights=np.ones(n_households), - metadata={"fit_scope": detected_scope}, - ) - - # Cache the sim on the state so the evaluator can reuse it later. - state.sim_service = sim - return state diff --git a/backend/services/db_service.py b/backend/services/db_service.py deleted file mode 100644 index af22a4c..0000000 --- a/backend/services/db_service.py +++ /dev/null @@ -1,157 +0,0 @@ -"""SQLModel ORM queries against policy_data.db.""" - -from sqlmodel import Session, select - -from policyengine_us_data.db.create_database_tables import ( - Stratum, - StratumConstraint, - Target, -) - - -def batch_get_all_stratum_constraints(session: Session) -> dict[int, list[dict]]: - """Return all constraints grouped by stratum_id in a single query.""" - all_constraints = session.exec(select(StratumConstraint)).all() - result: dict[int, list[dict]] = {} - for c in all_constraints: - result.setdefault(c.stratum_id, []).append({ - "variable": c.constraint_variable, - "operation": c.operation, - "value": c.value, - }) - return result - - -def get_target_provenance( - session: Session, - target_id: int, -) -> dict | None: - """Full target metadata including stratum constraints.""" - target = session.get(Target, target_id) - if target is None: - return None - - constraints = session.exec( - select(StratumConstraint).where( - StratumConstraint.stratum_id == target.stratum_id - ) - ).all() - - return { - "target_id": target.target_id, - "variable": target.variable, - "value": target.value, - "period": target.period, - "source": target.source, - "tolerance": target.tolerance, - "notes": target.notes, - "active": target.active, - "stratum_id": target.stratum_id, - "constraints": [ - { - "variable": c.constraint_variable, - "operation": c.operation, - "value": c.value, - } - for c in constraints - ], - } - - -def search_targets( - session: Session, - pattern: str, - active_only: bool = True, -) -> list[dict]: - """Search targets by variable name pattern.""" - stmt = select(Target).where(Target.variable.like(f"%{pattern}%")) - if active_only: - stmt = stmt.where(Target.active == True) # noqa: E712 - targets = session.exec(stmt).all() - return [ - { - "target_id": t.target_id, - "variable": t.variable, - "value": t.value, - "period": t.period, - "stratum_id": t.stratum_id, - "source": t.source, - "active": t.active, - } - for t in targets - ] - - -def get_stratum_detail( - session: Session, - stratum_id: int, -) -> dict | None: - """Stratum with constraints, children, and attached targets.""" - stratum = session.get(Stratum, stratum_id) - if stratum is None: - return None - - constraints = session.exec( - select(StratumConstraint).where( - StratumConstraint.stratum_id == stratum_id - ) - ).all() - - children = session.exec( - select(Stratum).where(Stratum.parent_stratum_id == stratum_id) - ).all() - - targets = session.exec( - select(Target).where( - Target.stratum_id == stratum_id, - Target.active == True, # noqa: E712 - ) - ).all() - - return { - "stratum_id": stratum.stratum_id, - "parent_stratum_id": stratum.parent_stratum_id, - "notes": stratum.notes, - "constraints": [ - { - "variable": c.constraint_variable, - "operation": c.operation, - "value": c.value, - } - for c in constraints - ], - "children": [ - {"stratum_id": ch.stratum_id, "notes": ch.notes} - for ch in children - ], - "targets": [ - { - "target_id": t.target_id, - "variable": t.variable, - "value": t.value, - "period": t.period, - "active": t.active, - } - for t in targets - ], - } - - -def get_target_constraints( - session: Session, - stratum_id: int, -) -> list[dict]: - """Return all constraints for a stratum.""" - constraints = session.exec( - select(StratumConstraint).where( - StratumConstraint.stratum_id == stratum_id - ) - ).all() - return [ - { - "variable": c.constraint_variable, - "operation": c.operation, - "value": c.value, - } - for c in constraints - ] diff --git a/backend/services/fit_artifacts.py b/backend/services/fit_artifacts.py deleted file mode 100644 index 2869a36..0000000 --- a/backend/services/fit_artifacts.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Local mirror of `policyengine_us_data.fit_weights.artifacts`. - -Mirrored from upstream policyengine-us-data PRs #1043 / #1046 (merged -2026-05-21), which centralized Stage 3 artifact filenames into a typed -catalog. Those changes aren't in PyPI 1.115.4 yet; the moment the team -cuts a release that includes the `fit_weights` module, drop this file and -swap callers to: - - from policyengine_us_data.fit_weights.artifacts import ( - fit_artifacts_for_scope, - ) - -Two scopes are published per Stage 3 fit: - -- **regional** — district / state-level fits. Filenames have no prefix. -- **national** — single national-scope fit. Filenames are `national_*`. - -Previously our loader only knew the regional names; a national run would -silently miss its diagnostics. The catalog makes the scope explicit. -""" - -from __future__ import annotations - -from dataclasses import dataclass - -REGIONAL_SCOPE = "regional" -NATIONAL_SCOPE = "national" -SCOPES = (REGIONAL_SCOPE, NATIONAL_SCOPE) - - -@dataclass(frozen=True) -class ScopedArtifacts: - scope: str - weights: str - geography: str - run_config: str - diagnostics: str - epoch_log: str - - def as_dict(self) -> dict[str, str]: - return { - "weights": self.weights, - "geography": self.geography, - "run_config": self.run_config, - "diagnostics": self.diagnostics, - "epoch_log": self.epoch_log, - } - - -REGIONAL_ARTIFACTS = ScopedArtifacts( - scope=REGIONAL_SCOPE, - weights="calibration_weights.npy", - geography="geography_assignment.npz", - run_config="unified_run_config.json", - diagnostics="unified_diagnostics.csv", - epoch_log="calibration_log.csv", -) - -NATIONAL_ARTIFACTS = ScopedArtifacts( - scope=NATIONAL_SCOPE, - weights="national_calibration_weights.npy", - geography="national_geography_assignment.npz", - run_config="national_unified_run_config.json", - diagnostics="national_unified_diagnostics.csv", - epoch_log="national_calibration_log.csv", -) - - -def artifacts_for_scope(scope: str) -> ScopedArtifacts: - if scope == REGIONAL_SCOPE: - return REGIONAL_ARTIFACTS - if scope == NATIONAL_SCOPE: - return NATIONAL_ARTIFACTS - raise ValueError(f"Unknown fit scope: {scope!r}") diff --git a/backend/services/geo_utils.py b/backend/services/geo_utils.py deleted file mode 100644 index 3116e30..0000000 --- a/backend/services/geo_utils.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Utilities for parsing congressional district GEOIDs and readable names.""" - -import numpy as np - -STATE_FIPS_TO_NAME: dict[int, str] = { - 1: "Alabama", 2: "Alaska", 4: "Arizona", 5: "Arkansas", - 6: "California", 8: "Colorado", 9: "Connecticut", 10: "Delaware", - 11: "District of Columbia", 12: "Florida", 13: "Georgia", 15: "Hawaii", - 16: "Idaho", 17: "Illinois", 18: "Indiana", 19: "Iowa", - 20: "Kansas", 21: "Kentucky", 22: "Louisiana", 23: "Maine", - 24: "Maryland", 25: "Massachusetts", 26: "Michigan", 27: "Minnesota", - 28: "Mississippi", 29: "Missouri", 30: "Montana", 31: "Nebraska", - 32: "Nevada", 33: "New Hampshire", 34: "New Jersey", 35: "New Mexico", - 36: "New York", 37: "North Carolina", 38: "North Dakota", 39: "Ohio", - 40: "Oklahoma", 41: "Oregon", 42: "Pennsylvania", 44: "Rhode Island", - 45: "South Carolina", 46: "South Dakota", 47: "Tennessee", 48: "Texas", - 49: "Utah", 50: "Vermont", 51: "Virginia", 53: "Washington", - 54: "West Virginia", 55: "Wisconsin", 56: "Wyoming", - 60: "American Samoa", 66: "Guam", 69: "Northern Mariana Islands", - 72: "Puerto Rico", 78: "U.S. Virgin Islands", -} - -STATE_NAME_TO_FIPS: dict[str, int] = {v: k for k, v in STATE_FIPS_TO_NAME.items()} - -# States with a single at-large congressional district -AT_LARGE_STATES: set[int] = {2, 10, 11, 30, 38, 46, 50, 56} - -STATE_FIPS_TO_ABBREV: dict[int, str] = { - 1: "AL", 2: "AK", 4: "AZ", 5: "AR", 6: "CA", 8: "CO", 9: "CT", - 10: "DE", 11: "DC", 12: "FL", 13: "GA", 15: "HI", 16: "ID", - 17: "IL", 18: "IN", 19: "IA", 20: "KS", 21: "KY", 22: "LA", - 23: "ME", 24: "MD", 25: "MA", 26: "MI", 27: "MN", 28: "MS", - 29: "MO", 30: "MT", 31: "NE", 32: "NV", 33: "NH", 34: "NJ", - 35: "NM", 36: "NY", 37: "NC", 38: "ND", 39: "OH", 40: "OK", - 41: "OR", 42: "PA", 44: "RI", 45: "SC", 46: "SD", 47: "TN", - 48: "TX", 49: "UT", 50: "VT", 51: "VA", 53: "WA", 54: "WV", - 55: "WI", 56: "WY", 60: "AS", 66: "GU", 69: "MP", 72: "PR", - 78: "VI", -} - - -def parse_cd_geoids(cd_geoid: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Extract state FIPS codes from congressional district GEOID strings. - - CD GEOIDs are stored as strings like '4213' (state 42, district 13) - or '621' (state 6, district 21). The state FIPS is everything except - the last two characters (the district number is always two digits, - but may lack a leading zero in the string representation). - - Args: - cd_geoid: String array of CD GEOIDs (e.g., ['4213', '621', '104']). - - Returns: - Tuple of (cd_geoid_int, state_fips) as integer arrays. - cd_geoid_int: The GEOID as an integer (e.g., 4213, 621, 104). - state_fips: The state FIPS code (e.g., 42, 6, 1). - """ - if len(cd_geoid) == 0: - return np.array([], dtype=np.int64), np.array([], dtype=np.int32) - - cd_geoid_int = np.array([int(x) for x in cd_geoid], dtype=np.int64) - state_fips = (cd_geoid_int // 100).astype(np.int32) - - return cd_geoid_int, state_fips - - -def state_name(fips: int) -> str: - """Return the state name for a FIPS code.""" - return STATE_FIPS_TO_NAME.get(fips, f"Unknown ({fips})") - - -def state_abbrev(fips: int) -> str: - """Return the state abbreviation for a FIPS code.""" - return STATE_FIPS_TO_ABBREV.get(fips, f"?{fips}") - - -def runtime_dataset_bundle_for( - geo_level: str | None, - geographic_id, - *, - available: "frozenset[str] | None" = None, - federal_fallback: str = "enhanced_cps_2024.h5", -) -> str: - """Like ``dataset_bundle_for`` but aware of what the run actually - publishes. If the conventional bundle (e.g. ``states/CA.h5``) isn't - in ``available``, fall back to the federal bundle so the dashboard - label matches the file that actually holds this target's weights. - - When ``available`` is None the function behaves like the raw - canonical mapping; pass a set from ``published_bundles`` to make it - run-aware. - """ - proposed = dataset_bundle_for(geo_level, geographic_id) - if not available: - return proposed - if proposed in available: - return proposed - if federal_fallback in available: - return federal_fallback - return proposed - - -def dataset_bundle_for(geo_level: str | None, geographic_id) -> str: - """Map a target's geography to the calibrated dataset file that holds - its calibrated weights, mirroring the per-state / per-district pipeline - builds in policyengine-us-data. - - Examples: - national, US → 'national/US.h5' - state, '6' → 'states/CA.h5' - district, 612 → 'districts/CA-12.h5' - - Used both for display (the existing 'Dataset' column on /targets) and - for the dataset_file filter — different bundles include different - target subsets, and the per-state h5 builds are calibrated against - their own slice of the targets table. - """ - if not geo_level or geo_level == "national": - return "national/US.h5" - gid_str = "" if geographic_id is None else str(geographic_id) - if not gid_str or gid_str == "nan": - return "—" - try: - n = int(float(gid_str)) - except (TypeError, ValueError): - n = None - if geo_level == "state": - if n is not None and n in STATE_FIPS_TO_ABBREV: - return f"states/{STATE_FIPS_TO_ABBREV[n]}.h5" - return f"states/{gid_str}.h5" - if geo_level == "district": - if n is not None: - state_fips = n // 100 - dist = n % 100 - code = STATE_FIPS_TO_ABBREV.get(state_fips, str(state_fips)) - return f"districts/{code}-{dist:02d}.h5" - return f"districts/{gid_str}.h5" - return str(geo_level) - - -def _ordinal(n: int) -> str: - """Return ordinal string for an integer (1 -> '1st', 2 -> '2nd', etc.).""" - if 11 <= n % 100 <= 13: - return f"{n}th" - suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th") - return f"{n}{suffix}" - - -def cd_display_name(cd_geoid_int: int) -> str: - """Return a readable name for a congressional district GEOID. - - Examples: - 4213 -> "Pennsylvania's 13th congressional district" - 621 -> "California's 21st congressional district" - 101 -> "Alabama's at-large congressional district" - 1101 -> "District of Columbia's at-large congressional district" - """ - fips = cd_geoid_int // 100 - district_num = cd_geoid_int % 100 - sname = STATE_FIPS_TO_NAME.get(fips, f"State {fips}") - - if fips in AT_LARGE_STATES or district_num == 1 and fips in AT_LARGE_STATES: - return f"{sname}'s at-large congressional district" - - return f"{sname}'s {_ordinal(district_num)} congressional district" - - -def geo_display_name(geo_level: str, geographic_id: str) -> str: - """Return a readable name for any geographic level + ID. - - Args: - geo_level: 'national', 'state', or 'district' - geographic_id: 'US', state FIPS string, or CD GEOID string - - Returns: - Readable name like 'National', 'California', or - "California's 21st congressional district" - """ - if geo_level == "national": - return "National" - - if geo_level == "state": - try: - fips = int(geographic_id) - return state_name(fips) - except (ValueError, TypeError): - return f"State {geographic_id}" - - if geo_level == "district": - try: - cd_int = int(geographic_id) - return cd_display_name(cd_int) - except (ValueError, TypeError): - return f"District {geographic_id}" - - return str(geographic_id) - - -def list_states() -> list[dict]: - """Return all states with FIPS, name, and abbreviation.""" - return [ - {"fips": fips, "name": name, "abbrev": STATE_FIPS_TO_ABBREV.get(fips, "")} - for fips, name in sorted(STATE_FIPS_TO_NAME.items()) - if fips <= 56 # exclude territories - ] - - -def list_districts_for_state(state_fips: int) -> list[dict]: - """Return all congressional districts for a state. - - Note: this returns a generic list based on known at-large states. - For actual district enumeration, query the calibration package's cd_geoid array. - """ - sname = state_name(state_fips) - if state_fips in AT_LARGE_STATES: - cd_int = state_fips * 100 + 1 - return [{"cd_geoid": cd_int, "name": f"{sname}'s at-large congressional district"}] - return [] diff --git a/backend/services/loader.py b/backend/services/loader.py deleted file mode 100644 index 30481c2..0000000 --- a/backend/services/loader.py +++ /dev/null @@ -1,427 +0,0 @@ -"""Load all calibration artifacts and compute derived fields at startup.""" - -import logging -import os -from pathlib import Path - -import numpy as np -import pandas as pd -from sqlalchemy import create_engine - -from policyengine_us_data.calibration.unified_calibration import ( - load_calibration_package, - load_target_config, -) - -from backend.services.geo_utils import parse_cd_geoids -from backend.services.sim_service import SimService -from backend.state import AppState - -logger = logging.getLogger(__name__) - -HF_ARTIFACTS = { - "package_path": "calibration_package.pkl", - "weights_path": "calibration_weights.npy", - "db_path": "policy_data.db", - "dataset_path": "source_imputed_stratified_extended_cps.h5", - "cal_log_path": "calibration_log.csv", - "diagnostics_path": "unified_diagnostics.csv", - "target_config_path": "target_config.yaml", -} - - -def _ensure_artifacts( - repo_id: str, - repo_type: str, - prefix: str, - cache_root: str = ".artifacts", -) -> dict: - """Download a run's artifacts from HuggingFace, cached per (repo, prefix). - - Returns a config dict mapping artifact keys to resolved local paths. - Caches under ///. - """ - from huggingface_hub import hf_hub_download - - repo_slug = repo_id.replace("/", "__") - cache = Path(cache_root) / repo_slug / prefix - cache.mkdir(parents=True, exist_ok=True) - resolved: dict = {} - - for key, hf_filename in HF_ARTIFACTS.items(): - dest = cache / hf_filename - if dest.exists(): - logger.info("Found cached %s: %s", key, dest) - resolved[key] = str(dest) - continue - - hf_path = f"{prefix}/{hf_filename}" - logger.info( - "Downloading %s from %s/%s ...", key, repo_id, hf_path - ) - try: - hf_hub_download( - repo_id=repo_id, - filename=hf_path, - repo_type=repo_type, - local_dir=str(cache), - ) - except Exception as exc: - logger.warning("Could not download %s: %s", hf_path, exc) - resolved[key] = None - continue - - nested = cache / prefix / hf_filename - if nested.exists() and not dest.exists(): - nested.rename(dest) - try: - (cache / prefix).rmdir() - except OSError: - pass - - resolved[key] = str(dest) if dest.exists() else None - if dest.exists(): - size_mb = dest.stat().st_size / 1e6 - logger.info(" Downloaded %s (%.1f MB)", hf_filename, size_mb) - - return resolved - - -def load_run( - repo_id: str, - repo_type: str, - prefix: str, - cache_root: str = ".artifacts", - dataset_id: str = "", -) -> AppState: - """Convenience entrypoint that resolves a run's artifacts and loads them.""" - config = _ensure_artifacts(repo_id, repo_type, prefix, cache_root) - state = load_all_artifacts(config) - state.dataset_id = dataset_id - state.run_id = prefix - return state - - -def load_all_artifacts(config: dict) -> AppState: - """Load every artifact and build the AppState. - - Args: - config: dict with keys: - package_path, weights_path, db_path, dataset_path, - cal_log_path (optional), diagnostics_path (optional), - target_config_path (optional) - """ - - # 1. Load calibration package - logger.info("Loading calibration package from %s", config["package_path"]) - package = load_calibration_package(config["package_path"]) - X_csr = package["X_sparse"] - targets_df = package["targets_df"] - target_names = package["target_names"] - initial_weights = package["initial_weights"] - cd_geoid = package.get("cd_geoid", np.array([])) - metadata = package.get("metadata", {}) - - n_targets, n_households = X_csr.shape - logger.info("Matrix shape: %d targets x %d households", n_targets, n_households) - - # 2. Load final weights - logger.info("Loading final weights from %s", config["weights_path"]) - final_weights = np.load(config["weights_path"]) - assert len(final_weights) == n_households, ( - f"Weight length {len(final_weights)} != matrix cols {n_households}" - ) - - # 3. Build CSC for column access - logger.info("Building CSC matrix for column access...") - X_csc = X_csr.tocsc() - - # 4. Initialize Microsimulation - logger.info("Initializing Microsimulation from %s", config["dataset_path"]) - from policyengine_us import Microsimulation - - sim = Microsimulation(dataset=config["dataset_path"]) - time_period = _detect_time_period(sim) - n_base = len(sim.calculate("household_id", map_to="household").values) - n_clones = n_households // n_base - logger.info( - "Base households: %d, clones: %d, total: %d", - n_base, n_clones, n_households, - ) - sim_service = SimService(sim, time_period, n_clones) - logger.info("Microsimulation ready, time_period=%d", time_period) - - # 5. Compute core household fields - logger.info("Computing household fields...") - hh_income = sim_service.calculate("spm_unit_net_income", map_to="household") - hh_threshold = sim_service.calculate( - "spm_unit_spm_threshold", map_to="household" - ) - - # 6. Derive g-weights, poverty flags, deciles - g_weights = final_weights / np.maximum(initial_weights, 1e-10) - in_poverty = hh_income < hh_threshold - income_decile = pd.qcut( - hh_income, 10, labels=False, duplicates="drop" - ) - - cd_geoid_int, state_fips = parse_cd_geoids(cd_geoid) - - households_df = pd.DataFrame({ - "household_idx": np.arange(n_households), - "income": hh_income.astype(np.float32), - "spm_threshold": hh_threshold.astype(np.float32), - "in_poverty": in_poverty, - "initial_weight": initial_weights.astype(np.float32), - "final_weight": final_weights.astype(np.float32), - "g_weight": g_weights.astype(np.float32), - "state": state_fips, - "cd_geoid": cd_geoid_int, - "income_decile": income_decile.astype(np.int8), - }) - logger.info("households_df: %d rows", len(households_df)) - - # 7. Enrich targets with error metrics and loss contribution - logger.info("Enriching targets with diagnostics...") - targets_enriched = _enrich_targets( - targets_df, target_names, X_csr, final_weights, initial_weights, in_poverty - ) - logger.info("targets_enriched: %d rows", len(targets_enriched)) - - # 8. Connect to targets database - db_engine = None - if config.get("db_path") and Path(config["db_path"]).exists(): - db_engine = create_engine(f"sqlite:///{config['db_path']}") - logger.info("Connected to policy_data.db at %s", config["db_path"]) - - # 8b. Join provenance fields (source / period / tolerance / notes) from - # policy_data.db into targets_enriched. The pkl's targets_df doesn't - # carry these — they're the calibration team's "what is this target, - # where did it come from" annotations. - if db_engine is not None and "target_id" in targets_enriched.columns: - try: - prov = pd.read_sql( - "SELECT target_id, source, period, tolerance, notes FROM targets", - db_engine, - ) - targets_enriched = targets_enriched.merge( - prov, on="target_id", how="left", suffixes=("", "_db"), - ) - # Prefer DB period over pkl's if both exist - if "period_db" in targets_enriched.columns: - targets_enriched["period"] = targets_enriched["period_db"].fillna( - targets_enriched.get("period") - ) - targets_enriched = targets_enriched.drop(columns=["period_db"]) - n_with_source = int(targets_enriched["source"].notna().sum()) - logger.info( - "Joined %d target provenance rows (source/period/tolerance/notes)", - n_with_source, - ) - except Exception: - logger.exception("Could not join target provenance from DB") - - # 9. Load target config and mark included/excluded - target_config = None - tc_path = config.get("target_config_path") - target_config_text: str | None = None - if tc_path and Path(tc_path).exists(): - target_config = load_target_config(tc_path) - try: - target_config_text = Path(tc_path).read_text(encoding="utf-8") - except OSError as exc: - logger.warning("Could not read raw target_config yaml: %s", exc) - targets_enriched = _apply_included_flag(targets_enriched, target_config) - logger.info( - "Target config: %d included, %d excluded", - targets_enriched["included"].sum(), - (~targets_enriched["included"]).sum(), - ) - # Recompute loss_contribution over included targets only - targets_enriched = _recompute_loss_for_included(targets_enriched) - else: - targets_enriched["included"] = True - logger.warning("No target config found — all targets marked as included") - - # 10. Batch-query constraints and add domain/additional columns - if db_engine is not None: - targets_enriched = _add_constraint_columns(targets_enriched, db_engine) - logger.info("Added domain and additional_constraints columns") - - # 11. Load optional CSVs - cal_log = _load_csv(config.get("cal_log_path")) - diagnostics_csv = _load_csv(config.get("diagnostics_path")) - - return AppState( - X_csr=X_csr, - X_csc=X_csc, - targets_df=targets_df, - target_names=target_names, - initial_weights=initial_weights, - cd_geoid=cd_geoid, - metadata=metadata, - final_weights=final_weights, - g_weights=g_weights, - targets_enriched=targets_enriched, - households_df=households_df, - sim_service=sim_service, - db_engine=db_engine, - cal_log=cal_log, - diagnostics_csv=diagnostics_csv, - target_config=target_config, - target_config_text=target_config_text, - time_period=time_period, - n_targets=n_targets, - n_households=n_households, - ) - - -def _detect_time_period(sim) -> int: - """Detect time period from the dataset.""" - raw_keys = sim.dataset.load_dataset()["household_id"] - if isinstance(raw_keys, dict): - return int(next(iter(raw_keys))) - return 2024 - - -def _enrich_targets( - targets_df: pd.DataFrame, - target_names: list[str], - X_csr, - final_weights: np.ndarray, - initial_weights: np.ndarray, - in_poverty: np.ndarray, -) -> pd.DataFrame: - """Add estimate, rel_error, loss_contribution, contributor counts.""" - enriched = targets_df.copy() - enriched["target_name"] = target_names[: len(enriched)] - - estimates = X_csr.dot(final_weights) - target_values = enriched["value"].values - - rel_errors = np.where( - np.abs(target_values) > 0, - (estimates - target_values) / np.abs(target_values), - 0.0, - ) - - enriched["estimate"] = estimates - enriched["rel_error"] = rel_errors - enriched["abs_rel_error"] = np.abs(rel_errors) - - # Loss contribution: each target's share of total squared relative error - squared_errors = rel_errors ** 2 - total_loss = squared_errors.sum() - if total_loss > 0: - enriched["loss_contribution"] = squared_errors / total_loss - else: - enriched["loss_contribution"] = 0.0 - - # Contributor counts (no poverty-specific metrics) - n_contributors = np.zeros(len(enriched), dtype=np.int32) - for i in range(len(enriched)): - n_contributors[i] = len(X_csr[i, :].nonzero()[1]) - enriched["n_contributors"] = n_contributors - - return enriched - - -def _apply_included_flag( - targets_enriched: pd.DataFrame, - target_config: dict, -) -> pd.DataFrame: - """Mark each target as included or excluded based on target_config rules.""" - include_rules = target_config.get("include", []) - exclude_rules = target_config.get("exclude", []) - - if not include_rules and not exclude_rules: - targets_enriched["included"] = True - return targets_enriched - - if include_rules: - mask = _match_rules(targets_enriched, include_rules) - else: - mask = np.ones(len(targets_enriched), dtype=bool) - - if exclude_rules: - drop = _match_rules(targets_enriched, exclude_rules) - mask &= ~drop - - targets_enriched["included"] = mask - return targets_enriched - - -def _match_rules(targets_df: pd.DataFrame, rules: list[dict]) -> np.ndarray: - """Build a boolean mask matching any of the given rules. - - Reimplements unified_calibration._match_rules for use at the app level. - """ - mask = np.zeros(len(targets_df), dtype=bool) - for rule in rules: - rule_mask = targets_df["variable"] == rule["variable"] - if "geo_level" in rule: - rule_mask = rule_mask & (targets_df["geo_level"] == rule["geo_level"]) - if "domain_variable" in rule: - rule_mask = rule_mask & ( - targets_df["domain_variable"] == rule["domain_variable"] - ) - mask |= rule_mask.values - return mask - - -def _recompute_loss_for_included(targets_enriched: pd.DataFrame) -> pd.DataFrame: - """Recompute loss_contribution scoped to included targets only.""" - included = targets_enriched["included"].values - sq_errors = targets_enriched["rel_error"].values ** 2 - total_included_loss = sq_errors[included].sum() - if total_included_loss > 0: - loss = np.where(included, sq_errors / total_included_loss, 0.0) - else: - loss = np.zeros(len(targets_enriched)) - targets_enriched["loss_contribution"] = loss - return targets_enriched - - -GEOGRAPHIC_VARS = {"state_fips", "congressional_district_geoid", "ucgid_str"} -ADDITIONAL_VARS = {"tax_unit_is_filer"} - - -def _add_constraint_columns( - targets_enriched: pd.DataFrame, - db_engine, -) -> pd.DataFrame: - """Add domain and additional_constraints columns. - - Domain comes from the package's domain_variable column (internally - consistent with the package data). Full constraint details from the - DB are only reliable when the DB and package are from the same build. - """ - # domain_variable from the package is the GROUP_CONCAT of - # non-geographic constraint variable names. Use it directly. - targets_enriched["domain"] = targets_enriched["domain_variable"].apply( - lambda v: str(v) if pd.notna(v) else None - ) - - # For additional_constraints, check if the target name contains - # tax_unit_is_filer. The target_name encodes all constraints in - # its [...] suffix, so we can parse it from there. - def _extract_additional(target_name: str) -> str | None: - if not isinstance(target_name, str): - return None - parts = [] - if "tax_unit_is_filer" in target_name: - parts.append("tax_unit_is_filer == 1") - return ", ".join(parts) or None - - targets_enriched["additional_constraints"] = targets_enriched["target_name"].apply( - _extract_additional - ) - return targets_enriched - - -def _load_csv(path: str | None) -> pd.DataFrame | None: - """Load a CSV if the path exists.""" - if path and Path(path).exists(): - logger.info("Loading CSV from %s", path) - return pd.read_csv(path) - return None diff --git a/backend/services/matrix_ops.py b/backend/services/matrix_ops.py deleted file mode 100644 index 8b79ada..0000000 --- a/backend/services/matrix_ops.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Sparse matrix operations for calibration diagnostics.""" - -import numpy as np -import pandas as pd -import scipy.sparse as sp - - -def get_target_contributors( - X_csr: sp.csr_matrix, - target_idx: int, -) -> np.ndarray: - """Return household indices that contribute to a target (CSR row slice).""" - return X_csr[target_idx, :].nonzero()[1] - - -def get_target_contributions( - X_csr: sp.csr_matrix, - target_idx: int, - final_weights: np.ndarray, -) -> pd.DataFrame: - """Return contributing households with raw and weighted values.""" - row = X_csr[target_idx, :] - cols = row.nonzero()[1] - raw_values = np.array(row[:, cols].todense()).flatten() - weighted_values = raw_values * final_weights[cols] - return pd.DataFrame({ - "household_idx": cols, - "raw_value": raw_values, - "weighted_value": weighted_values, - }) - - -def get_household_targets( - X_csc: sp.csc_matrix, - household_idx: int, -) -> np.ndarray: - """Return target indices this household contributes to (CSC column slice).""" - return X_csc[:, household_idx].nonzero()[0] - - -def get_household_attributions( - X_csc: sp.csc_matrix, - household_idx: int, - final_weights: np.ndarray, -) -> pd.DataFrame: - """Return targets this household contributes to, with contribution values.""" - col = X_csc[:, household_idx] - rows = col.nonzero()[0] - raw_values = np.array(col[rows, :].todense()).flatten() - weighted_values = raw_values * final_weights[household_idx] - return pd.DataFrame({ - "target_idx": rows, - "raw_value": raw_values, - "weighted_value": weighted_values, - }) - - -def compute_error_decomposition( - X_csr: sp.csr_matrix, - target_idx: int, - target_value: float, - initial_weights: np.ndarray, - final_weights: np.ndarray, -) -> dict: - """Three-number decomposition: raw_sum, initial_est, final_est vs target.""" - row = X_csr[target_idx, :] - raw_sum = float(row.sum()) - initial_est = float(row.dot(initial_weights)) - final_est = float(row.dot(final_weights)) - - if target_value != 0: - raw_ratio = raw_sum / target_value - if abs(raw_ratio - 1) > 0.5: - diagnosis = ( - f"raw_sum is {(raw_ratio - 1):+.0%} off target — " - "variable values or coverage wrong in source data" - ) - elif abs(initial_est / target_value - 1) < 0.15 and abs( - final_est / target_value - 1 - ) > 0.15: - diagnosis = ( - "Initial weights close to target but final drifted — " - "other constraints pulled weights away" - ) - else: - diagnosis = "See raw_sum, initial_estimate, final_estimate for details" - else: - diagnosis = "Target value is zero" - - return { - "target_value": target_value, - "raw_sum": raw_sum, - "initial_estimate": initial_est, - "final_estimate": final_est, - "diagnosis": diagnosis, - } - - -def compute_concentration( - X_csr: sp.csr_matrix, - target_idx: int, - final_weights: np.ndarray, -) -> dict: - """Top 1% and 5% weighted contribution share for a target.""" - row = X_csr[target_idx, :] - cols = row.nonzero()[1] - if len(cols) == 0: - return {"top_1pct_share": 0.0, "top_5pct_share": 0.0} - - raw_values = np.array(row[:, cols].todense()).flatten() - contributions = raw_values * final_weights[cols] - total = contributions.sum() - if total == 0: - return {"top_1pct_share": 0.0, "top_5pct_share": 0.0} - - sorted_contrib = np.sort(contributions)[::-1] - n = len(sorted_contrib) - top_1 = sorted_contrib[: max(1, n // 100)].sum() / total - top_5 = sorted_contrib[: max(1, n // 20)].sum() / total - return {"top_1pct_share": float(top_1), "top_5pct_share": float(top_5)} diff --git a/backend/services/registry.py b/backend/services/registry.py deleted file mode 100644 index 54fe0a3..0000000 --- a/backend/services/registry.py +++ /dev/null @@ -1,61 +0,0 @@ -"""LRU registry of loaded calibration runs. - -Each (dataset_id, run_id) maps to a fully-loaded AppState. Loading is -expensive (~30s + memory for sparse matrices + Microsimulation), so we -cache a small number and evict in LRU order. -""" - -from __future__ import annotations - -import logging -import threading -from collections import OrderedDict - -from backend.services import runs as runs_service -from backend.state import AppState - -logger = logging.getLogger(__name__) - - -class RunRegistry: - """Bounded LRU cache of loaded AppStates, keyed by (dataset_id, run_id).""" - - def __init__(self, max_size: int = 3): - self.max_size = max_size - self._cache: OrderedDict[tuple[str, str], AppState] = OrderedDict() - # Coarse lock: prevent two concurrent loads of the same run from - # racing. Acceptable since loads are infrequent. - self._lock = threading.Lock() - - def get(self, dataset_id: str, run_id: str) -> AppState: - key = (dataset_id, run_id) - with self._lock: - if key in self._cache: - self._cache.move_to_end(key) - return self._cache[key] - dataset = runs_service.get_dataset(dataset_id) - logger.info( - "Loading run %s/%s (layout=%s, cache miss; %d/%d loaded)", - dataset_id, run_id, dataset.layout, - len(self._cache), self.max_size, - ) - if dataset.layout in {"staging", "root", "staging-root"}: - from backend.services.dataset_loader import ( - load_run_from_dataset, - ) - state = load_run_from_dataset(dataset, run_id) - else: - from backend.services.loader import load_run # lazy: heavy deps - state = load_run( - dataset.repo_id, dataset.repo_type, run_id, - dataset_id=dataset_id, - ) - self._cache[key] = state - while len(self._cache) > self.max_size: - evicted_key, _ = self._cache.popitem(last=False) - logger.info("Evicted run %s/%s from cache", *evicted_key) - return state - - def loaded_keys(self) -> list[tuple[str, str]]: - with self._lock: - return list(self._cache.keys()) diff --git a/backend/services/runs.py b/backend/services/runs.py deleted file mode 100644 index ed920bb..0000000 --- a/backend/services/runs.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Discovery of available calibration runs on HuggingFace. - -A "dataset" is a HuggingFace repository plus a layout convention. A "run" is -one published calibration build whose artifacts live under some prefix in the -repo. Four layouts are supported today: - -- **flat**: repo// (the legacy sandbox repo) -- **staging**: repo/staging// (the canonical us-data repo) -- **root**: repo/ (current production files) -- **staging-root**: repo/staging/ (current staging snapshot) - -Each layout declares which artifact filenames must be present for a candidate -prefix to count as a real run, since the two repos publish different files. -""" - -from __future__ import annotations - -import logging -import os -from dataclasses import dataclass -from functools import lru_cache - -from huggingface_hub import HfApi - -logger = logging.getLogger(__name__) - -# Default required files per layout. Override per-dataset if a particular -# repo publishes a different file set. -DEFAULT_REQUIRED_FILES = { - "flat": ("calibration_package.pkl", "calibration_weights.npy"), - "staging": ("policy_data.db",), # at least the targets DB; an .h5 picked at load time - "root": ("policy_data.db",), - "staging-root": ("policy_data.db",), -} - -CURRENT_STAGING_RUN_ID = "staging" - - -@dataclass(frozen=True) -class DatasetConfig: - """A dataset = an HF repo plus a layout convention.""" - - id: str - label: str - repo_id: str - repo_type: str = "model" - layout: str = "flat" # flat, staging, root, staging-root - # Files that must exist under a prefix for it to be considered a run. - required_files: tuple[str, ...] = () - # For the staging layout: which h5 dataset to load when this dataset is - # selected. The pipeline publishes several (cps, enhanced_cps, ...). - primary_h5: str = "enhanced_cps_2024.h5" - - def effective_required_files(self) -> tuple[str, ...]: - if self.required_files: - return self.required_files - defaults = DEFAULT_REQUIRED_FILES.get(self.layout, ()) - # Dataset layouts also require the configured primary h5 to be present. - if self.layout in {"staging", "root", "staging-root"} and self.primary_h5: - return defaults + (self.primary_h5,) - return defaults - - -DEFAULT_DATASETS: list[DatasetConfig] = [ - # us-cps (sandbox / pkl mode) was retired once the canonical us-data - # publication started shipping unified_diagnostics.csv with full target - # coverage. Re-add it here if you need to compare against a pkl-snapshot - # run; the flat-layout loader is still wired up. - DatasetConfig( - id="us-data", - label="US Data - Enhanced CPS", - repo_id="PolicyEngine/policyengine-us-data", - layout="staging", - primary_h5="enhanced_cps_2024.h5", - ), - DatasetConfig( - id="us-data-production", - label="US Data - Production Enhanced CPS", - repo_id="PolicyEngine/policyengine-us-data", - layout="root", - primary_h5="enhanced_cps_2024.h5", - ), - DatasetConfig( - id="us-data-current-staging", - label="US Data - Current Staging", - repo_id="PolicyEngine/policyengine-us-data", - layout="staging-root", - primary_h5="source_imputed_stratified_extended_cps.h5", - ), - DatasetConfig( - id="us-data-cps", - label="US Data - CPS", - repo_id="PolicyEngine/policyengine-us-data", - layout="staging", - primary_h5="cps_2024.h5", - ), - DatasetConfig( - id="us-data-small-enhanced-cps", - label="US Data - Small Enhanced CPS", - repo_id="PolicyEngine/policyengine-us-data", - layout="staging", - primary_h5="small_enhanced_cps_2024.h5", - ), -] - - -@dataclass(frozen=True) -class RunInfo: - dataset_id: str - run_id: str # the prefix path within the repo - label: str - last_modified: str | None = None - - -def list_datasets() -> list[DatasetConfig]: - return list(DEFAULT_DATASETS) - - -def get_dataset(dataset_id: str) -> DatasetConfig: - for d in DEFAULT_DATASETS: - if d.id == dataset_id: - return d - raise KeyError(f"Unknown dataset_id: {dataset_id}") - - -def _group_flat(files: list[str]) -> dict[str, set[str]]: - """For flat layout: prefix = first path segment, files = direct children.""" - by_prefix: dict[str, set[str]] = {} - for path in files: - if "/" not in path: - continue - prefix, _, rest = path.partition("/") - if "/" in rest: # only files directly under / - continue - by_prefix.setdefault(prefix, set()).add(rest) - return by_prefix - - -def _group_staging(files: list[str]) -> dict[str, set[str]]: - """For staging layout, group files by run. - - Supports two artifact shapes the us-data team uses interchangeably: - - 1. **GHA / flat staging**: ``staging//policy_data.db``, - ``staging//enhanced_cps_2024.h5`` at the run root. - 2. **Versioned-release nested**: ``staging//calibration/policy_data.db`` - and ``staging//datasets/enhanced_cps_2024.h5``. - - Both layouts get the *same* set of logical filenames so the - discovery + required-files check in :func:`list_runs` works - uniformly. The actual on-disk download location is resolved later - by :func:`_resolve_staging_file_paths`. - """ - by_prefix: dict[str, set[str]] = {} - for path in files: - if not path.startswith("staging/"): - continue - parts = path.split("/", 2) - if len(parts) < 3: - continue - run_id = parts[1] - rest = parts[2] - if "/" not in rest: - # Flat staging — file sits at the run root. - by_prefix.setdefault(run_id, set()).add(rest) - continue - # Nested staging — recognise the locations the pipeline uses. - head, _, tail = rest.partition("/") - if head == "calibration" and tail == "policy_data.db": - by_prefix.setdefault(run_id, set()).add("policy_data.db") - elif head == "datasets" and tail.endswith(".h5") and "/" not in tail: - by_prefix.setdefault(run_id, set()).add(tail) - return by_prefix - - -def _group_staging_root(files: list[str]) -> dict[str, set[str]]: - """Expose the top-level ``staging/`` snapshot as one synthetic run.""" - names: set[str] = set() - for path in files: - if path == "staging/calibration/policy_data.db": - names.add("policy_data.db") - continue - if ( - path.startswith("staging/calibration/") - and path.endswith(".h5") - and path.count("/") == 2 - ): - names.add(path.rsplit("/", 1)[-1]) - return {CURRENT_STAGING_RUN_ID: names} if names else {} - - -def _resolve_staging_file_paths( - repo_id: str, - run_id: str, - logical_names: list[str], -) -> dict[str, str]: - """Map each logical filename (e.g. ``policy_data.db``, ``enhanced_cps_2024.h5``) - to its actual path on HF, probing both flat and nested layouts. - - Returns only entries that were found; callers should treat a missing - key as 'this run doesn't publish that file'. - """ - api = HfApi() - files = set(api.list_repo_files(repo_id, repo_type="model")) - prefix = f"staging/{run_id}" - resolved: dict[str, str] = {} - candidates: dict[str, list[str]] = { - "policy_data.db": [ - f"{prefix}/policy_data.db", - f"{prefix}/calibration/policy_data.db", - ], - } - for ln in logical_names: - for cand in candidates.get(ln, [f"{prefix}/{ln}", f"{prefix}/datasets/{ln}"]): - if cand in files: - resolved[ln] = cand - break - return resolved - - -def _resolve_staging_root_file_paths( - repo_id: str, - logical_names: list[str], -) -> dict[str, str]: - """Resolve logical artifact names in the current top-level staging snapshot.""" - api = HfApi() - files = set(api.list_repo_files(repo_id, repo_type="model")) - resolved: dict[str, str] = {} - candidates: dict[str, list[str]] = { - "policy_data.db": ["staging/calibration/policy_data.db"], - } - for name in logical_names: - for candidate in candidates.get(name, [f"staging/calibration/{name}"]): - if candidate in files: - resolved[name] = candidate - break - return resolved - - -@lru_cache(maxsize=8) -def list_runs(dataset_id: str) -> tuple[RunInfo, ...]: - """Discover runs for a dataset. Cached; restart to refresh from HF.""" - dataset = get_dataset(dataset_id) - api = HfApi() - try: - files = api.list_repo_files( - repo_id=dataset.repo_id, repo_type=dataset.repo_type - ) - except Exception: - logger.exception("Failed to list HF repo %s", dataset.repo_id) - return () - - if dataset.layout == "flat": - by_prefix = _group_flat(files) - elif dataset.layout == "staging": - by_prefix = _group_staging(files) - elif dataset.layout == "root": - by_prefix = {"main": {path for path in files if "/" not in path}} - elif dataset.layout == "staging-root": - by_prefix = _group_staging_root(files) - else: - logger.error("Unknown layout %r for dataset %s", dataset.layout, dataset_id) - return () - - required = dataset.effective_required_files() - runs: list[RunInfo] = [] - for prefix, names in sorted(by_prefix.items(), reverse=True): - if not all(req in names for req in required): - continue - runs.append( - RunInfo( - dataset_id=dataset_id, - run_id=prefix, - label="Current staging" if dataset.layout == "staging-root" else prefix, - last_modified=None, # cheap; populate lazily if needed - ) - ) - return tuple(runs) - - -def storage_prefix(dataset: DatasetConfig, run_id: str) -> str: - """The path prefix within the repo where this run's files live.""" - if dataset.layout == "staging": - return f"staging/{run_id}" - if dataset.layout == "staging-root": - return "staging" - if dataset.layout == "root": - return "" - return run_id - - -def default_selection() -> tuple[str, str] | None: - ds = os.environ.get("DEFAULT_DATASET") - run = os.environ.get("DEFAULT_RUN") - if ds and run: - return ds, run - return None diff --git a/backend/services/sim_service.py b/backend/services/sim_service.py deleted file mode 100644 index bf49496..0000000 --- a/backend/services/sim_service.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Wrapper around Microsimulation.calculate() with result caching. - -All results are tiled to clone-level length so they align with -X_sparse columns, weights, and cd_geoid arrays. -""" - -from typing import Any - -import numpy as np - - -class SimService: - """Caches sim.calculate() results tiled to clone-level.""" - - def __init__(self, sim: Any, time_period: int, n_clones: int) -> None: - self._sim = sim - self._time_period = time_period - self._n_clones = n_clones - self._cache: dict[str, np.ndarray] = {} - - def calculate( - self, - variable: str, - map_to: str = "household", - ) -> np.ndarray: - """Compute a variable and tile to clone-level length. - - The Microsimulation returns one value per base household. - This method tiles the result n_clones times so it aligns - with the calibration package arrays (n_records * n_clones). - """ - key = f"{variable}:{map_to}" - if key not in self._cache: - base_values = ( - self._sim.calculate(variable, self._time_period, map_to=map_to) - .values.astype(np.float64) - ) - self._cache[key] = np.tile(base_values, self._n_clones) - return self._cache[key] - - def get_variable_entity(self, variable: str) -> str: - """Return the native entity key for a variable.""" - var_def = self._sim.tax_benefit_system.variables.get(variable) - if var_def is None: - raise ValueError(f"Unknown variable: {variable}") - return var_def.entity.key - - def get_formula_dependencies(self, variable: str) -> list[str]: - """Return the variable's adds/subtracts dependencies.""" - var_def = self._sim.tax_benefit_system.variables.get(variable) - if var_def is None: - return [] - deps: list[str] = [] - adds = getattr(var_def, "adds", None) - subtracts = getattr(var_def, "subtracts", None) - if isinstance(adds, list): - deps.extend(adds) - if isinstance(subtracts, list): - deps.extend(subtracts) - return deps - - def variable_exists(self, variable: str) -> bool: - return variable in self._sim.tax_benefit_system.variables diff --git a/backend/services/stratum_evaluator.py b/backend/services/stratum_evaluator.py deleted file mode 100644 index ec465c1..0000000 --- a/backend/services/stratum_evaluator.py +++ /dev/null @@ -1,510 +0,0 @@ -"""Compute target estimates from a calibrated dataset (no X matrix). - -Two scopes: - -1. `evaluate_targets()` — historical entry point used by the dataset-mode - loader. Handles geographic-only constraints across a pandas DataFrame. - -2. `evaluate_signature()` — single-target evaluator used by the inventory - endpoint to fill PE aggregates for authored-only rows. Adds support for - any constraint variable that PolicyEngine can map to the household - entity (e.g. `snap > 0`, `medicaid_enrolled == 1`, - `tax_unit_is_filer == 1`), and treats `is_count` targets as weighted - counts of matching households rather than weighted variable sums. - -Out of scope (both functions): person-level constraints whose meaning -breaks under household aggregation (e.g. `age < 5` to count under-5 -individuals — household-mapping sums ages, not bodies). The pipeline's -`UnifiedMatrixBuilder` handles those correctly via entity mapping. -""" - -from __future__ import annotations - -import logging -import operator as op_module - -import numpy as np -import pandas as pd - -logger = logging.getLogger(__name__) - - -_OPS = { - "==": op_module.eq, "!=": op_module.ne, - ">": op_module.gt, ">=": op_module.ge, - "<": op_module.lt, "<=": op_module.le, -} - - -def _apply_op(op_str: str, arr, cval): - """Apply a comparison operator, handling the multi-value `in` case. - - SOI-style constraints encode multi-value filters as - ``filing_status in JOINT|SURVIVING_SPOUSE`` (pipe-delimited). Our - standard _OPS dict only covers scalar comparisons, so handle `in` - explicitly here. - """ - if op_str == "in": - choices = [c.strip() for c in str(cval).split("|") if c.strip()] - # Try numeric coercion of choices; fall back to string compare. - try: - numeric = [float(c) for c in choices] - return np.isin(arr, numeric) - except (TypeError, ValueError): - return np.isin(np.asarray(arr).astype(str), choices) - op = _OPS.get(op_str) - if op is None: - raise ValueError(f"unknown operator {op_str!r}") - try: - target_val = float(cval) - except (TypeError, ValueError): - target_val = cval - return op(arr, target_val) - -# Constraint variables whose household-mapping makes the mask meaningless -# (person-level continuous values like age). evaluate_signature() refuses -# these only in the household-level path; the entity-aware path -# (_evaluate_at_entity) handles them correctly by working at the variable's -# native entity. -_ENTITY_ONLY_VARS = { - "age", - "person_id", - "spm_unit_id", - "tax_unit_id", - "family_id", -} - - -_NON_HOUSEHOLD_ENTITIES = ("person", "tax_unit", "spm_unit", "family", "marital_unit") - - -def _tbs_from(sim) -> "object | None": - """Return the TBS from either a SimService wrapper or a raw Microsim.""" - if sim is None: - return None - return getattr(getattr(sim, "_sim", sim), "tax_benefit_system", None) - - -def _variable_entity(sim, variable: str) -> str | None: - tbs = _tbs_from(sim) - if tbs is None: - return None - v = tbs.variables.get(variable) - return v.entity.key if v is not None else None - - -def _calculate_at(sim, variable: str, entity: str, period: int): - """Compute a variable mapped to a specific entity and return the - MicroSeries (carries both ``.values`` and ``.weights`` at that entity). - - Supports both the SimService wrapper (pkl mode) and raw Microsim - (dataset mode). Returns None on failure. - """ - target_sim = getattr(sim, "_sim", sim) - try: - return target_sim.calculate(variable, map_to=entity, period=period) - except Exception as exc: - logger.debug("calculate(%s, map_to=%s) failed: %s", variable, entity, exc) - return None - - -def _is_geographic_only(constraints: list[str]) -> bool: - """A row is 'geographic-only' if all constraints are state/district.""" - if not constraints: - return True - geo_prefixes = ( - "state_fips ", "congressional_district_geoid ", "ucgid_str ", - ) - return all(c.startswith(geo_prefixes) for c in constraints) - - -def _evaluate_variable(sim, variable: str, period: int) -> np.ndarray | None: - """Calculate a variable at household level, weighted. Returns the weighted - values per household, or None if the variable can't be evaluated.""" - try: - series = sim.calculate(variable, map_to="household", period=period) - except Exception as exc: - logger.debug("Could not calculate %s for %s: %s", variable, period, exc) - return None - # MicroSeries × weights = weighted values per household - return series.values * series.weights - - -class EvalCache: - """Holds per-state-of-the-world caches across many evaluate_signature - calls. Built from the loaded AppState so it transparently supports both - the sandbox path (SimService — tiled to clone-level) and the dataset - path (raw Microsimulation — MicroSeries with weights). - """ - - def __init__(self, state): - self.state = state - self.period = int(state.time_period) - self._raw: dict[tuple[str, int], np.ndarray | None] = {} - self._weights: np.ndarray | None = None - - # --- Internal: source of truth for one variable, evaluated to a 1-D - # per-household array aligned with `weights()` below. - def _calculate(self, variable: str) -> np.ndarray | None: - svc = self.state.sim_service - if svc is None: - return None - try: - if hasattr(svc, "_sim"): # SimService wrapper (sandbox path) - return np.asarray(svc.calculate(variable, map_to="household")) - # Otherwise treat as raw Microsimulation - series = svc.calculate(variable, map_to="household", period=self.period) - if self._weights is None and hasattr(series, "weights"): - self._weights = np.asarray(series.weights) - return np.asarray(series.values) - except Exception as exc: - logger.debug("calculate(%s) failed: %s", variable, exc) - return None - - def raw(self, variable: str, period: int | None = None) -> np.ndarray | None: - """Per-household raw values for `variable`. period is informational - only — the underlying sim is fixed to one time period for now.""" - key = (variable, period or self.period) - if key not in self._raw: - self._raw[key] = self._calculate(variable) - return self._raw[key] - - def weighted(self, variable: str, period: int | None = None) -> np.ndarray | None: - """Per-household weighted values: raw * household_weight.""" - arr = self.raw(variable, period) - w = self.weights() - if arr is None or w is None or len(arr) != len(w): - return None - return arr * w - - def weights(self) -> np.ndarray | None: - # The loaded AppState already has the calibrated final_weights tiled. - w = getattr(self.state, "final_weights", None) - if w is not None and len(w) > 0: - return np.asarray(w) - if self._weights is not None: - return self._weights - return self.raw("household_weight") - - def state_fips(self) -> np.ndarray | None: - return self.raw("state_fips") - - def cd_geoid(self) -> np.ndarray | None: - return self.raw("congressional_district_geoid") - - -def _evaluate_at_entity( - variable: str, - geo_level: str | None, - geographic_id: str | None, - constraints, - is_count: bool, - sim, - entity: str, - period: int, -) -> tuple[float | None, str]: - """Entity-aware evaluation: works at the target variable's native - entity (person / tax_unit / spm_unit / family) rather than household. - - Required for targets like population counts with age-range constraints, - or SOI tax-unit aggregates with filing_status / AGI bracket constraints - — household-level evaluation gets these wrong because the constraint - can't be meaningfully aggregated to household. - """ - # Per-entity MicroSeries cache: name → MicroSeries - series_cache: dict[str, object] = {} - - def calc(v: str): - if v not in series_cache: - series_cache[v] = _calculate_at(sim, v, entity, period) - return series_cache[v] - - target_series = calc(variable) - if target_series is None: - return None, f"target {variable!r} not evaluable at entity={entity}" - target_vals = np.asarray(target_series.values) - target_weights = np.asarray(target_series.weights) - n = len(target_vals) - - # Geo mask at this entity. PE projects state_fips / cd_geoid from - # household down to person / tax_unit / spm_unit automatically. - if geo_level in (None, "national"): - mask = np.ones(n, dtype=bool) - elif geo_level == "state": - sf_s = calc("state_fips") - if sf_s is None: - return None, f"state_fips not evaluable at {entity}" - sf = np.asarray(sf_s.values) - try: - mask = sf == int(geographic_id) - except (TypeError, ValueError): - mask = sf.astype(str) == str(geographic_id) - elif geo_level == "district": - cd_s = calc("congressional_district_geoid") - if cd_s is None: - return None, f"cd_geoid not evaluable at {entity}" - cd = np.asarray(cd_s.values) - try: - mask = cd.astype(int) == int(geographic_id) - except (TypeError, ValueError): - mask = cd.astype(str) == str(geographic_id) - else: - return None, f"geo_level={geo_level!r} not supported" - - # Constraint masks at this entity - for cvar, op_str, cval in constraints or (): - c_s = calc(cvar) - if c_s is None: - return None, f"constraint {cvar!r} not evaluable at {entity}" - arr = np.asarray(c_s.values) - try: - m = _apply_op(op_str, arr, cval) - except Exception as exc: - return None, f"could not evaluate {cvar} {op_str} {cval}: {exc}" - mask = mask & m - - # Empty-mask: the PE dataset has zero records matching this slice - # (common at district × multi-filter combinations). Return None instead - # of 0 so the UI can distinguish "no data to estimate" from "PE estimate - # is genuinely zero." A spurious 0 here looks like a -100% error vs the - # target and pollutes the ranked views. - if not mask.any(): - return None, f"no records match constraints at entity={entity}" - - if is_count: - return ( - float(target_weights[mask].sum()), - f"count at entity={entity}", - ) - return ( - float((target_vals[mask] * target_weights[mask]).sum()), - f"dollar at entity={entity}", - ) - - -def evaluate_signature( - variable: str, - geo_level: str | None, - geographic_id: str | None, - constraints: list[tuple[str, str, str]] | tuple, - is_count: bool, - cache: EvalCache, - period: int | None = None, -) -> tuple[float | None, str]: - """Single-target evaluator. Returns (estimate, eval_note). - - Routes to the entity-aware path automatically when the target - variable's native entity is not household, or when any constraint is - on a variable whose household-mapping would be wrong (age, etc.). - """ - period = period or cache.period - - # Decide working entity: target variable's entity, unless a constraint - # forces a more-granular entity. Default to household when TBS lookup - # is unavailable so behavior matches the legacy path. - sim = cache.state.sim_service - target_entity = _variable_entity(sim, variable) or "household" - forced_person = any( - (c[0] if isinstance(c, (list, tuple)) else "") in _ENTITY_ONLY_VARS - for c in (constraints or ()) - ) - if forced_person and target_entity == "household": - target_entity = "person" - - if target_entity in _NON_HOUSEHOLD_ENTITIES: - return _evaluate_at_entity( - variable, geo_level, geographic_id, constraints, - is_count, sim, target_entity, period, - ) - - # --- Household-level path (legacy / fast) --- - # --- Geographic mask --- - if geo_level == "national" or geo_level is None: - geo_mask = None # full set - elif geo_level == "state": - sf = cache.state_fips() - if sf is None: - return None, "state_fips not evaluable on dataset" - try: - geo_mask = sf == int(geographic_id) - except (TypeError, ValueError): - geo_mask = sf.astype(str) == str(geographic_id) - elif geo_level == "district": - cd = cache.cd_geoid() - if cd is None: - return None, "congressional_district_geoid not evaluable on dataset" - try: - geo_mask = cd.astype(int) == int(geographic_id) - except (TypeError, ValueError): - geo_mask = cd.astype(str) == str(geographic_id) - else: - return None, f"geo_level={geo_level!r} not supported" - - # --- Non-geographic constraint masks --- - constraint_mask = None - for cvar, op_str, cval in constraints or (): - if cvar in _ENTITY_ONLY_VARS: - return None, f"constraint on {cvar} requires entity-level evaluation" - arr = cache.raw(cvar, period) - if arr is None: - return None, f"constraint variable {cvar!r} not available" - try: - m = _apply_op(op_str, arr, cval) - except Exception as exc: - return None, f"could not evaluate {cvar} {op_str} {cval}: {exc}" - constraint_mask = m if constraint_mask is None else (constraint_mask & m) - - # Combine masks - if geo_mask is None and constraint_mask is None: - combined = None # full set - elif geo_mask is None: - combined = constraint_mask - elif constraint_mask is None: - combined = geo_mask - else: - combined = geo_mask & constraint_mask - - # Empty-mask: see notes in _evaluate_at_entity — return None, not 0, so - # "no records match" reads as missing rather than a real estimate. - if combined is not None and not combined.any(): - return None, "no records match constraints at entity=household" - - # --- Aggregate --- - if is_count: - weights = cache.weights() - if weights is None: - return None, "household_weight not evaluable" - if combined is None: - return float(weights.sum()), "count: full population" - return float(weights[combined].sum()), "count: weighted household sum where mask" - else: - weighted = cache.weighted(variable, period) - if weighted is None: - return None, f"target variable {variable!r} not available" - if combined is None: - return float(weighted.sum()), "dollar: full population" - return float(weighted[combined].sum()), "dollar: weighted sum where mask" - - -def _parse_constraint(c) -> tuple[str, str, str] | None: - """Coerce one constraint into a (var, op, value) tuple. - - Loader rows store constraints as either pre-parsed tuples or strings of - the form ``"var op value"`` (e.g. ``"filing_status == JOINT"``). We need - both forms because the diagnostics-CSV join and the DB load take - different paths into the same dataframe. - """ - if isinstance(c, (list, tuple)) and len(c) >= 3: - return str(c[0]), str(c[1]), str(c[2]) - s = str(c).strip() - # Order matters: longer operators first so "==" beats "=", " in " stays - # word-bounded so it doesn't match substrings inside a variable name. - for op, sep in ( - ("==", "=="), ("!=", "!="), (">=", ">="), ("<=", "<="), - (">", ">"), ("<", "<"), - ("in", " in "), - ): - idx = s.find(f" {sep.strip()} ") if op != "in" else s.find(sep) - if idx == -1 and op != "in": - # Allow no-space form for scalar ops (e.g. "x>=5") - idx = s.find(op) - if idx <= 0: - continue - if idx == -1: - continue - left = s[:idx].strip() - # For 'in', sep includes the surrounding spaces. - skip = len(sep) if op == "in" else len(op) - right = s[idx + skip:].strip().lstrip(" ").lstrip(op).strip() - if left: - return left, op, right - return None - - -def _looks_like_count_target(variable: str) -> bool: - """Detect count-style targets by naming convention. - - PE doesn't expose a "is this a population count" flag we can read off - the variable, so we fall back to the convention that population counts - end in ``_count`` (e.g. ``household_count``, ``tax_unit_count``). For - these, we want a weighted sum of the boolean mask, not of the variable - itself. - """ - return variable.endswith("_count") - - -def evaluate_targets( - targets_df: pd.DataFrame, - sim, - default_period: int = 2024, -) -> pd.DataFrame: - """Fill estimate / rel_error / abs_rel_error for as many rows as we can. - - Uses ``evaluate_signature`` so constraint targets (filing_status, - income brackets, has_children, etc.) get evaluated against the loaded - Microsimulation, not just geographic aggregates. Anything that's still - unevaluable (person-level continuous constraints, missing variables) - keeps a NaN estimate and an explanatory ``eval_note``. - """ - from types import SimpleNamespace - - out = targets_df.copy() - out["eval_note"] = "" - - # EvalCache expects a state-like object exposing sim_service + - # final_weights + time_period. The pkl-mode loader passes its real - # AppState; the dataset-mode loader passes a raw Microsimulation, so we - # synthesize a minimal one here. - state_like = SimpleNamespace( - sim_service=sim, - final_weights=np.array([]), - time_period=default_period, - ) - cache = EvalCache(state_like) - - estimates = np.full(len(out), np.nan, dtype=np.float64) - notes = np.array([""] * len(out), dtype=object) - - for i, (_, row) in enumerate(out.iterrows()): - variable = row["variable"] - # Ignore row["period"] — that's metadata about the source year of - # target_value (e.g. SOI 2022). The PE estimate must be at the - # dataset's period (2024 for the current h5); asking PE for any - # other year silently returns zeros instead of erroring. - period = default_period - geo_level = row.get("geo_level") - gid = row.get("geographic_id") - raw_constraints = row.get("constraints") or [] - parsed = [_parse_constraint(c) for c in raw_constraints] - if any(p is None for p in parsed): - notes[i] = f"unparseable constraint in {raw_constraints!r}" - continue - constraints = [c for c in parsed if c is not None] - is_count = _looks_like_count_target(variable) - - est, note = evaluate_signature( - variable=variable, - geo_level=geo_level, - geographic_id=gid, - constraints=constraints, - is_count=is_count, - cache=cache, - period=period, - ) - if est is None: - notes[i] = note - continue - estimates[i] = est - - out["estimate"] = estimates - out["eval_note"] = notes - target_values = out["value"].to_numpy(dtype=np.float64) - with np.errstate(divide="ignore", invalid="ignore"): - rel = np.where( - np.abs(target_values) > 0, - (estimates - target_values) / np.abs(target_values), - np.nan, - ) - out["rel_error"] = rel - out["abs_rel_error"] = np.abs(rel) - return out diff --git a/backend/state.py b/backend/state.py deleted file mode 100644 index 0e3d2e0..0000000 --- a/backend/state.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Application state holding all loaded calibration artifacts.""" - -from dataclasses import dataclass, field -from typing import Any, Optional - -import numpy as np -import pandas as pd -import scipy.sparse as sp -from fastapi import HTTPException, Query, Request -from sqlalchemy.engine import Engine - - -@dataclass -class AppState: - """Immutable container for all loaded calibration data for one run. - - Two load paths populate this: - - **pkl mode** (legacy / sandbox): full X matrix, weights from pkl/npy. - - **dataset mode** (canonical staging): targets read from policy_data.db, - estimates computed by evaluating PE variables on the published h5. - X_csr / X_csc / initial_weights stay empty in this mode. - """ - - # -- From calibration_package.pkl (pkl mode) or empty (dataset mode) -- - X_csr: sp.csr_matrix = field(default_factory=lambda: sp.csr_matrix((0, 0))) - X_csc: sp.csc_matrix = field(default_factory=lambda: sp.csc_matrix((0, 0))) - targets_df: pd.DataFrame = field(default_factory=pd.DataFrame) - target_names: list[str] = field(default_factory=list) - initial_weights: np.ndarray = field(default_factory=lambda: np.array([])) - cd_geoid: np.ndarray = field(default_factory=lambda: np.array([])) - metadata: dict = field(default_factory=dict) - - # -- From calibration_weights.npy -- - final_weights: np.ndarray = field(default_factory=lambda: np.array([])) - - # -- Computed at startup -- - g_weights: np.ndarray = field(default_factory=lambda: np.array([])) - targets_enriched: pd.DataFrame = field(default_factory=pd.DataFrame) - households_df: pd.DataFrame = field(default_factory=pd.DataFrame) - - # -- Services -- - sim_service: Any = None # SimService instance - db_engine: Optional[Engine] = None - - # -- Optional calibration logs -- - cal_log: Optional[pd.DataFrame] = None - diagnostics_csv: Optional[pd.DataFrame] = None - - # -- Target config -- - target_config: Optional[dict] = None - target_config_text: Optional[str] = None # raw yaml, preserves comments - - # -- Derived scalars -- - time_period: int = 2024 - n_targets: int = 0 - n_households: int = 0 - - # -- Provenance: which run produced this state -- - dataset_id: str = "" - run_id: str = "" - - -def get_state( - request: Request, - dataset: str | None = Query( - None, description="Dataset id (e.g. 'us-cps'). Falls back to DEFAULT_DATASET env." - ), - run: str | None = Query( - None, description="Run id (HF prefix). Falls back to DEFAULT_RUN env." - ), -) -> "AppState": - """FastAPI dependency that resolves a run and returns its AppState. - - Resolution order: - 1. ?dataset & ?run query params - 2. DEFAULT_DATASET / DEFAULT_RUN env vars (set at backend startup) - 3. 400 error - """ - from backend.services import runs as runs_service - - if dataset is None or run is None: - fallback = runs_service.default_selection() - if fallback is None: - raise HTTPException( - status_code=400, - detail=( - "No run selected. Pass ?dataset=&run= query params, or " - "set DEFAULT_DATASET and DEFAULT_RUN env vars." - ), - ) - dataset = dataset or fallback[0] - run = run or fallback[1] - - registry = request.app.state.registry - try: - return registry.get(dataset, run) - except KeyError as exc: - raise HTTPException(status_code=404, detail=str(exc)) - except Exception as exc: - raise HTTPException( - status_code=500, detail=f"Failed to load run {dataset}/{run}: {exc}" - ) diff --git a/frontend/app/analysis/page.tsx b/frontend/app/analysis/page.tsx deleted file mode 100644 index fa9e936..0000000 --- a/frontend/app/analysis/page.tsx +++ /dev/null @@ -1,582 +0,0 @@ -"use client"; - -import { useEffect, useMemo, useState } from "react"; -import { - Badge, - Card, - CardContent, - CardHeader, - CardTitle, - Spinner, - Stack, - Text, - Title, - formatNumber, -} from "@policyengine/ui-kit"; - -import { AppShell } from "@/components/layout/app-shell"; -import { - useDomainBreakdown, - useDependencyTrace, - usePolicyEngineVariables, - useTargetConfigAudit, - type DomainBreakdownRow, - type DependencyNode, - type PolicyEngineVariable, -} from "@/lib/api/hooks/use-analysis"; - -function pct(value: number | null | undefined, digits = 1): string { - if (value === null || value === undefined || !Number.isFinite(value)) { - return "—"; - } - return `${(value * 100).toFixed(digits)}%`; -} - -function num(value: number | null | undefined): string { - if (value === null || value === undefined || !Number.isFinite(value)) { - return "—"; - } - return formatNumber(value); -} - -function Kpi({ - label, - value, - hint, -}: { - label: string; - value: string; - hint?: string; -}) { - return ( -
- - {label} - -
{value}
- {hint && ( - - {hint} - - )} -
- ); -} - -function InlineLoading({ label }: { label: string }) { - return ( -
- - {label} -
- ); -} - -function RuleText({ rule }: { rule: Record }) { - return ( - - {Object.entries(rule) - .map(([key, value]) => `${key}=${value}`) - .join(", ")} - - ); -} - -function TargetConfigAuditPanel({ variable }: { variable: string | null }) { - const audit = useTargetConfigAudit(variable); - const zeroRules = useMemo( - () => audit.data?.rules.filter((r) => r.status === "zero_match") ?? [], - [audit.data], - ); - - return ( - - -
- Target config audit - {variable && {variable}} -
- - - {!variable && ( - - Select a variable to audit matching target-config rules. - - )} - {audit.isLoading && ( - - )} - {audit.error && ( - Failed to load audit: {String(audit.error)} - )} - {audit.data && ( - -
- - - - -
- {audit.data.rules.length === 0 && ( - - No target-config rules matched this variable's target or domain - rows. - - )} - {zeroRules.length > 0 && ( -
- - - - - - - - - - {zeroRules.slice(0, 20).map((rule) => ( - - - - - - ))} - -
SectionIndexRule
{rule.section}{rule.index} - -
-
- )} - - )} - - - ); -} - -function DomainBreakdownPanel({ variable }: { variable: string | null }) { - const [domainVariable, setDomainVariable] = useState( - "adjusted_gross_income", - ); - const breakdown = useDomainBreakdown(variable, domainVariable); - - return ( - - -
- Domain breakdown - {variable && {variable}} -
- - -
- setDomainVariable(e.target.value)} - placeholder="Domain variable, e.g. adjusted_gross_income" - className="h-10 min-w-[260px] flex-1 rounded-md border border-border bg-background px-3 text-sm" - /> -
- {!variable && ( - - Showing calibration targets grouped by the selected domain variable. - Select a target variable to narrow the breakdown. - - )} - {breakdown.isLoading && ( - - )} - {breakdown.error && ( - - Failed to load domain breakdown: {String(breakdown.error)} - - )} - {breakdown.data && ( - -
- - - - -
- {breakdown.data.rows.length === 0 ? ( - - No targets found for this domain variable and selected output. - - ) : ( -
- - - - - - - - - - - - - - {breakdown.data.rows.map((row: DomainBreakdownRow) => ( - - - - - - - - - - ))} - -
Domain bucketTargetsIncludedEvaluatedMedian errorMax errorGeo
{row.bucket} - {num(row.target_count)} - - {num(row.included_target_count)} - - {num(row.evaluated_target_count)} - - {pct(row.median_abs_rel_error)} - - {pct(row.max_abs_rel_error)} - {row.geo_levels.join(", ")}
-
- )} - - )} - - - ); -} - -function nodeStatus(node: DependencyNode): string { - if (node.is_target_variable) return "target"; - if (node.is_domain_variable) return "domain"; - if (node.is_stored_input) return "stored"; - if (node.is_formula) return "formula"; - if (node.is_aggregate) return "aggregate"; - return "other"; -} - -function targetRole(node: DependencyNode): string { - if (node.direct_target_count > 0 && node.domain_target_count > 0) { - return "direct + domain"; - } - if (node.direct_target_count > 0) return "direct"; - if (node.domain_target_count > 0) return "domain"; - return "none"; -} - -function DependencyPanel({ - variable, - onVariableChange, -}: { - variable: string | null; - onVariableChange: (variable: string | null) => void; -}) { - const [search, setSearch] = useState(""); - const [nodeSearch, setNodeSearch] = useState(""); - const [nodeScope, setNodeScope] = useState("all"); - const [kindFilter, setKindFilter] = useState("all"); - const [targetFilter, setTargetFilter] = useState("all"); - const variableCatalog = usePolicyEngineVariables(search); - const trace = useDependencyTrace(variable); - useEffect(() => { - setNodeSearch(""); - setNodeScope("all"); - setKindFilter("all"); - setTargetFilter("all"); - }, [variable]); - const variableOptions = useMemo(() => { - const items = variableCatalog.data?.items ?? []; - const scored = [...items].sort((a, b) => { - const aTarget = a.is_target_variable || a.is_domain_variable ? 0 : 1; - const bTarget = b.is_target_variable || b.is_domain_variable ? 0 : 1; - if (aTarget !== bTarget) return aTarget - bTarget; - return a.name.localeCompare(b.name); - }); - return scored.slice(0, 80); - }, [variableCatalog.data]); - const filteredNodes = useMemo(() => { - const nodes = trace.data?.nodes ?? []; - const q = nodeSearch.trim().toLowerCase(); - return nodes.filter((node) => { - const status = nodeStatus(node); - if (nodeScope === "inputs" && !node.is_leaf && !node.is_stored_input) { - return false; - } - if (nodeScope === "non_inputs" && (node.is_leaf || node.is_stored_input)) { - return false; - } - if (kindFilter !== "all" && status !== kindFilter) { - return false; - } - if ( - targetFilter === "targeted" && - !node.is_target_variable && - !node.is_domain_variable - ) { - return false; - } - if ( - targetFilter === "untargeted" && - (node.is_target_variable || node.is_domain_variable) - ) { - return false; - } - if (q) { - const haystack = `${node.variable} ${node.label} ${node.entity ?? ""}`.toLowerCase(); - if (!haystack.includes(q)) { - return false; - } - } - return true; - }); - }, [kindFilter, nodeScope, nodeSearch, targetFilter, trace.data]); - - return ( - - -
- Variable explorer - {variable && {variable}} -
- - -
- setSearch(e.target.value)} - placeholder="Search any PolicyEngine-US variable, e.g. ca_income_tax" - className="h-10 min-w-[280px] flex-1 rounded-md border border-border bg-background px-3 text-sm" - /> - {variableCatalog.isLoading && ( - - )} - {variableCatalog.error && ( - - Failed to load variable catalog: {String(variableCatalog.error)} - - )} - {variableCatalog.data && ( -
- {variableOptions.map((item: PolicyEngineVariable) => ( - - ))} - {variableOptions.length === 0 && ( - - No variables match that search. - - )} -
- )} -
- {trace.isLoading && ( -
- - - Tracing formula dependencies for {variable}... - -
- )} - {trace.error && ( - Failed to trace {variable}: {String(trace.error)} - )} - {!variable && !trace.isLoading && ( - - Search for a variable and select it to inspect calibration targets, - dependencies, stored inputs, and propagation. - - )} - {trace.data && ( - -
- - - - - -
-
- setNodeSearch(e.target.value)} - placeholder="Filter traced variables" - className="h-10 min-w-[220px] flex-1 rounded-md border border-border bg-background px-3 text-sm" - /> - - - - - {num(filteredNodes.length)} / {num(trace.data.nodes.length)} nodes - -
-
- - - - - - - - - - - - - - - - - {filteredNodes.slice(0, 80).map((node) => ( - - - - - - - - - - - - - ))} - {filteredNodes.length === 0 && ( - - - - )} - -
DepthVariableEntityKindDepsTarget roleTargetsEvaluatedMedian errorMax error
{num(node.depth)}{node.variable}{node.entity ?? "—"} - {nodeStatus(node)} - - {num(node.dependency_count)} - {targetRole(node)} - {num(node.included_target_count)} / {num(node.target_count)} - - {num(node.evaluated_target_count)} - - {pct(node.median_abs_rel_error)} - - {pct(node.max_abs_rel_error)} -
- No traced nodes match these filters. -
-
- - )} - - - ); -} - -export default function AnalysisPage() { - const [selectedVariable, setSelectedVariable] = useState(null); - - return ( - - -
- Analyst readiness - - Reform-specific calibration coverage, propagation, and pipeline - checks for the selected run. - -
- - - - - - - - - ); -} diff --git a/frontend/app/api/microplex/budget-benchmarks/route.ts b/frontend/app/api/microplex/budget-benchmarks/route.ts deleted file mode 100644 index bddfb13..0000000 --- a/frontend/app/api/microplex/budget-benchmarks/route.ts +++ /dev/null @@ -1,274 +0,0 @@ -import { NextResponse } from "next/server"; - -const ROWS = [ - { - id: "american_family_act_2025", - title: "American Family Act 2025 CTC expansion", - policy_area: "Child Tax Credit", - benchmark_period: "2026 annual", - comparison_status: "live_model_no_third_party_score", - budget_effect_rule: "credit_delta_is_cost", - notes: - "No independent public budget score is attached for this bill. PolicyEngine has a published static analysis, but that is not a third-party benchmark and is intentionally excluded from the external comparison slot.", - external_estimates: [ - { - source: "CBO/JCT", - source_type: "official_score", - url: "https://www.congress.gov/bill/119th-congress/house-bill/2763", - estimate: null, - estimate_label: "No public CBO/JCT score found for H.R.2763 / S.1393.", - period: "not available", - }, - ], - }, - { - id: "working_parents_tax_relief_act_2026", - title: "Working Parents Tax Relief Act EITC enhancement", - policy_area: "Earned Income Tax Credit", - benchmark_period: "2026 annual", - comparison_status: "live_model_partial_external_context", - budget_effect_rule: "credit_delta_is_cost", - notes: - "Live microsim rows require the Python backend and a configured Microplex H5 artifact root.", - external_estimates: [ - { - source: "Thomson Reuters coverage", - source_type: "third_party_context", - url: "https://tax.thomsonreuters.com/news/bill-seeks-earned-income-tax-credit-boost-per-child-for-working-parents/", - estimate: null, - estimate_label: - "Third-party coverage found; no single budget score is attached in this catalog.", - period: "not available", - }, - { - source: "PolicyEngine policy page", - source_type: "published_model_result", - url: "https://www.policyengine.org/us/working-parents-tax-relief-act", - estimate: null, - estimate_label: "PolicyEngine analysis; not a CBO/JCT comparator.", - period: "2026+", - }, - ], - }, - { - id: "wyden_smith_ctc_2024", - title: "Wyden-Smith / TRAFWA CTC provisions", - policy_area: "Child Tax Credit", - benchmark_period: "2024 annual", - comparison_status: "live_model_with_third_party_score", - budget_effect_rule: "credit_delta_is_cost", - notes: - "This is a true third-party benchmark row when connected to the Python backend. The static fallback cannot run the live us-data and Microplex microsims.", - external_estimates: [ - { - source: "Joint Committee on Taxation", - source_type: "jct", - url: "https://waysandmeans.house.gov/wp-content/uploads/2024/01/Estimated-Revenue-Effects-of-H.R.-7024.pdf", - estimate: 10_700_000_000, - estimate_label: - "$10.7B 2024 cost for the combined CTC provisions, as reported in PolicyEngine's JCT comparison table.", - period: "2024", - comparable_to_live_annual_result: true, - }, - { - source: "Joint Committee on Taxation", - source_type: "jct", - url: "https://waysandmeans.house.gov/wp-content/uploads/2024/01/Estimated-Revenue-Effects-of-H.R.-7024.pdf", - estimate: 33_493_000_000, - estimate_label: - "$33.493B 2024-2033 revenue effect for the Tax Relief for Working Families line in JCX-3-24.", - period: "2024-2033", - comparable_to_live_annual_result: false, - }, - ], - }, - { - id: "kypa_ctc_2026", - title: "Keep Your Pay Act expanded CTC", - policy_area: "Child Tax Credit", - benchmark_period: "2026 annual", - comparison_status: "live_model_with_third_party_score", - budget_effect_rule: "credit_delta_is_cost", - notes: - "PWBM provides separable KYPA CTC estimates. Live dashboard comparison uses the 2026 annual CTC delta against PWBM's FY2027 line as a full-year fiscal proxy. PWBM's FY2026 line is much smaller because refundable credit timing shifts most TY2026 cost into FY2027. The PolicyEngine-US reform is an AFA-style approximation, so the newborn-bonus mechanics may not match PWBM exactly.", - external_estimates: [ - { - source: "Penn Wharton Budget Model", - source_type: "pwbm", - url: "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - estimate: 140_500_000_000, - estimate_label: - "$140.5B FY2027 revenue loss for KYPA's expanded Child Tax Credit provision. This is the closest full-year fiscal proxy for a calendar-year 2026 microsim.", - period: "FY2027 proxy for TY2026", - comparable_to_live_annual_result: true, - }, - { - source: "Penn Wharton Budget Model", - source_type: "pwbm", - url: "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - estimate: 2_500_000_000, - estimate_label: - "$2.5B FY2026 revenue loss. Included as timing context; not comparable to a full calendar-year 2026 tax microsim.", - period: "FY2026 timing context", - comparable_to_live_annual_result: false, - }, - { - source: "Penn Wharton Budget Model", - source_type: "pwbm", - url: "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - estimate: 1_261_600_000_000, - estimate_label: - "$1.2616T FY2026-2035 revenue loss for KYPA's expanded Child Tax Credit provision.", - period: "FY2026-2035", - comparable_to_live_annual_result: false, - }, - ], - }, - { - id: "kypa_childless_eitc_2026", - title: "Keep Your Pay Act childless-worker EITC", - policy_area: "Earned Income Tax Credit", - benchmark_period: "2026 annual", - comparison_status: "live_model_with_third_party_score", - budget_effect_rule: "credit_delta_is_cost", - notes: - "PWBM provides a separable childless-worker EITC estimate. The live comparison applies the same 2026 parameters described by PWBM and compares to the FY2027 line as a full-year fiscal proxy: minimum age 19, no maximum age, 15.3% phase-in and phase-out rates, about a $1,502 maximum credit, and about an $11,610 phase-out start.", - external_estimates: [ - { - source: "Penn Wharton Budget Model", - source_type: "pwbm", - url: "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - estimate: 7_200_000_000, - estimate_label: - "$7.2B FY2027 revenue loss for KYPA's childless-worker EITC expansion. This is the closest full-year fiscal proxy for a calendar-year 2026 microsim.", - period: "FY2027 proxy for TY2026", - comparable_to_live_annual_result: true, - }, - { - source: "Penn Wharton Budget Model", - source_type: "pwbm", - url: "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - estimate: 800_000_000, - estimate_label: - "$0.8B FY2026 revenue loss. Included as timing context; not comparable to a full calendar-year 2026 tax microsim.", - period: "FY2026 timing context", - comparable_to_live_annual_result: false, - }, - { - source: "Penn Wharton Budget Model", - source_type: "pwbm", - url: "https://budgetmodel.wharton.upenn.edu/p/2026-03-11-the-keep-your-pay-act-budgetary-and-distributional-effects/", - estimate: 63_800_000_000, - estimate_label: - "$63.8B FY2026-2035 revenue loss for KYPA's childless-worker EITC expansion.", - period: "FY2026-2035", - comparable_to_live_annual_result: false, - }, - ], - }, - { - id: "tcja_extension_2026_2035", - title: "TCJA individual provisions extension", - policy_area: "Federal individual income tax", - benchmark_period: "2026-2035", - comparison_status: "external_score_available_reform_not_wired", - budget_effect_rule: "full_budget_score", - notes: - "External benchmark is strong, but a matching live TCJA-extension reform preset is not wired yet.", - external_estimates: [ - { - source: "CBO/JCT", - source_type: "cbo_jct", - url: "https://www.policyengine.org/us/research/tcja-extension", - estimate: 3_877_600_000_000, - estimate_label: "$3.8776T cost over 2026-2035", - period: "2026-2035", - }, - { - source: "CRFB", - source_type: "third_party_score", - url: "https://www.policyengine.org/us/research/tcja-extension", - estimate: 3_830_000_000_000, - estimate_label: "$3.83T cost over 2026-2035", - period: "2026-2035", - }, - { - source: "PolicyEngine dynamic", - source_type: "published_model_result", - url: "https://www.policyengine.org/us/research/tcja-extension", - estimate: 3_885_500_000_000, - estimate_label: "$3.8855T cost over 2026-2035", - period: "2026-2035", - }, - ], - }, - { - id: "final_2025_reconciliation_tax", - title: "Final 2025 reconciliation individual income tax provisions", - policy_area: "Federal individual income tax", - benchmark_period: "2026-2035", - comparison_status: "external_score_available_reform_not_wired", - budget_effect_rule: "full_budget_score", - notes: - "PolicyEngine-US baseline now contains many OBBBA provisions, so live comparison needs an explicit counterfactual reform branch.", - external_estimates: [ - { - source: "PolicyEngine static analysis", - source_type: "published_model_result", - url: "https://www.policyengine.org/us/research/final-2025-reconciliation-tax", - estimate: 3_785_000_000_000, - estimate_label: "$3.785T cost over 2026-2035", - period: "2026-2035", - }, - { - source: "JCT JCX-26-25", - source_type: "jct", - url: "https://www.jct.gov/publications/2025/jcx-26-25/", - estimate: null, - estimate_label: - "Official JCT revenue estimate available; row-level match not wired.", - period: "2025 budget reconciliation", - }, - ], - }, -]; - -export const revalidate = 300; - -function liveUnavailable() { - return { - available: false, - reason: - "The deployed static fallback cannot run PolicyEngine microsims. Set NEXT_PUBLIC_API_URL to a Python backend for live us-data and Microplex values.", - reform: null, - period: null, - outcome_variable: null, - outcome_entity: null, - unit: null, - us_data: null, - microplex: null, - microplex_budget_effect_as_share_of_us_data: null, - budget_effect_gap: null, - }; -} - -export async function GET() { - return NextResponse.json({ - available: true, - runtime_seconds: 0, - generated_at_unix: Date.now() / 1000, - sign_convention: - "Positive budget effect means higher federal cost or lower federal revenue. For current live CTC/EITC rows, this equals the aggregate credit increase.", - comparison_caveat: - "This is the deployed static fallback. External benchmark rows are visible, but live us-data and Microplex microsim values require the Python backend.", - us_data_dataset: "not connected", - microplex_bundle: { - available: false, - artifact_id: null, - artifact_dir: null, - policyengine_dataset_path: null, - }, - rows: ROWS.map((row) => ({ ...row, live: liveUnavailable() })), - errors: [], - }); -} diff --git a/frontend/app/api/microplex/reform-comparison/route.ts b/frontend/app/api/microplex/reform-comparison/route.ts deleted file mode 100644 index bfa530b..0000000 --- a/frontend/app/api/microplex/reform-comparison/route.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { NextResponse } from "next/server"; - -const AVAILABLE_REFORMS = [ - { - id: "american_family_act_2025", - label: "American Family Act 2025 CTC expansion", - description: - "CTC expansion preset used for local us-data versus Microplex microsim checks.", - variable: "ctc", - entity: "tax_unit", - period: 2026, - unit: "USD", - source_url: "https://www.congress.gov/bill/119th-congress/house-bill/2763", - }, - { - id: "working_parents_tax_relief_act_2026", - label: "Working Parents Tax Relief Act EITC enhancement", - description: - "EITC enhancement preset used for local reform-sensitivity checks.", - variable: "eitc", - entity: "tax_unit", - period: 2026, - unit: "USD", - source_url: - "https://tax.thomsonreuters.com/news/bill-seeks-earned-income-tax-credit-boost-per-child-for-working-parents/", - }, - { - id: "halve_joint_eitc_phase_out_rate", - label: "Halve joint-filer EITC phase-out rate", - description: - "Small EITC policy perturbation for comparing dataset sensitivity.", - variable: "eitc", - entity: "tax_unit", - period: 2026, - unit: "USD", - }, -]; - -export const revalidate = 300; - -export async function GET(request: Request) { - const url = new URL(request.url); - const reformId = - url.searchParams.get("reform_id") ?? AVAILABLE_REFORMS[0]?.id ?? null; - const reform = - AVAILABLE_REFORMS.find((item) => item.id === reformId) ?? - AVAILABLE_REFORMS[0] ?? - null; - - return NextResponse.json({ - available: false, - reason: - "The deployed Vercel frontend is using static public Microplex artifacts. Live us-data versus Microplex microsim comparisons require a hosted Python backend with NEXT_PUBLIC_API_URL configured, or a local dashboard opened from the local frontend.", - runtime_seconds: 0, - period: reform?.period ?? 2026, - available_reforms: AVAILABLE_REFORMS, - reform, - microplex_bundle: { - artifact_id: null, - artifact_dir: null, - policyengine_dataset_path: null, - }, - us_data_dataset: "not connected", - outcomes: [], - }); -} diff --git a/frontend/app/api/microplex/route.ts b/frontend/app/api/microplex/route.ts deleted file mode 100644 index 3872edf..0000000 --- a/frontend/app/api/microplex/route.ts +++ /dev/null @@ -1,303 +0,0 @@ -import { NextResponse } from "next/server"; - -import { - LATEST_MICROPLEX_ARTIFACT_ID, - LATEST_MICROPLEX_NATIVE_SCORES_PATH, - LATEST_MICROPLEX_TARGET_DIAGNOSTICS_PATH, - latestHeadlinePatch, - latestNativeScores, - latestTargetDiagnosticsSummary, -} from "@/lib/microplex/latest-artifact"; - -const GITHUB_RAW = "https://raw.githubusercontent.com/PolicyEngine/microplex-us/main"; - -const PARITY_PATH = - "artifacts/live_pe_native_cps_puf_rich_broad_fixed_20260329/20260329T175330Z-057066af/pe_us_data_rebuild_parity.json"; -const REGRESSION_SUMMARY_PATH = - "artifacts/live_pe_us_data_rebuild_checkpoint_modelpass_regression_summary_20260410.json"; -const IRS_DRILLDOWN_PATH = - "artifacts/live_pe_us_data_rebuild_checkpoint_national_irs_other_drilldown_20260410.json"; -const RUN_LEVEL_TARGET_DIAGNOSTICS_PATH = "pe_native_target_diagnostics.json"; -const RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY = - "policyengine_native_target_diagnostics"; -const LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH = - "artifacts/pe_native_target_diagnostics_current.json"; - -const ARTIFACTS = { - parity: PARITY_PATH, - regression_summary: REGRESSION_SUMMARY_PATH, - irs_drilldown: IRS_DRILLDOWN_PATH, -}; - -const GENERATED_ARTIFACT_CONTRACT = [ - { - name: "full_target_diagnostics", - path_hint: RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - manifest_key: RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - legacy_static_dashboard_path: LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH, - producer: "build_us_pe_native_target_diagnostics_payload", - public_committed: false, - description: - "Full per-target PE-native rows saved inside each newer Microplex run bundle. The rows show Microplex aggregate estimates against target values, with us-data comparator fields when present.", - }, - { - name: "dashboard_payload", - path_hint: "artifacts/microplex_dashboard_current.json", - producer: "microplex-us-dashboard", - public_committed: false, - description: - "Living dashboard payload with score runs, logs, and target diagnostics.", - }, - { - name: "native_scores", - path_hint: "policyengine_native_scores.json", - producer: "compute_us_pe_native_scores", - public_committed: false, - description: "Compact broad native-loss summary for one artifact bundle.", - }, - { - name: "native_audit", - path_hint: "pe_us_data_rebuild_native_audit.json", - producer: "build_policyengine_us_data_rebuild_native_audit", - public_committed: false, - description: "Top family and target regressions plus support audit evidence.", - }, - { - name: "run_index", - path_hint: "run_index.duckdb", - producer: "append_us_microplex_run_index_entry", - public_committed: false, - description: "DuckDB index for querying target deltas across saved runs.", - }, -]; - -const TARGET_DIAGNOSTIC_ROW_FIELDS = [ - "target_id", - "family", - "in_loss", - "supported_by_microplex", - "baseline_dataset", - "candidate_dataset", - "baseline_label", - "candidate_label", - "target_value", - "us_data_aggregate", - "microplex_aggregate", - "us_data_absolute_error", - "microplex_absolute_error", - "us_data_relative_error", - "microplex_relative_error", - "delta_absolute_error", - "delta_relative_error", - "loss_contribution", -]; - -type JsonObject = Record; - -export const revalidate = 300; - -function asObject(value: unknown): JsonObject { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as JsonObject) - : {}; -} - -function scrub(value: unknown): unknown { - if (Array.isArray(value)) return value.map(scrub); - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value as JsonObject).map(([key, item]) => [key, scrub(item)]), - ); - } - if (typeof value === "number" && !Number.isFinite(value)) return null; - return value; -} - -function discoverConfiguredRunBundles() { - return { - artifact_root_env: "MICROPLEX_ARTIFACT_ROOTS", - single_artifact_root_env: "MICROPLEX_ARTIFACT_ROOT", - configured_artifact_roots: [], - missing_artifact_roots: [], - detected_run_bundle_count: 0, - detected_target_diagnostics_count: 0, - latest_run_bundle: null, - sampled_run_bundles: [], - }; -} - -async function fetchJson(path: string): Promise { - const response = await fetch(`${GITHUB_RAW}/${path}`, { - next: { revalidate }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch ${path}: ${response.status}`); - } - return asObject(await response.json()); -} - -export async function GET() { - try { - const [parity, regression, drilldown] = await Promise.all([ - fetchJson(PARITY_PATH), - fetchJson(REGRESSION_SUMMARY_PATH), - fetchJson(IRS_DRILLDOWN_PATH), - ]); - - const comparison = asObject(parity.comparison); - const policyengineHarness = asObject(comparison.policyengineHarness); - const baselineSlice = asObject(parity.baselineSlice); - const comparisonMetadata = asObject(baselineSlice.comparisonMetadata); - const tagSummaries = asObject(policyengineHarness.tag_summaries); - const allTargets = asObject(tagSummaries.all_targets); - const configuredRuns = discoverConfiguredRunBundles(); - const targetRowsAvailable = true; - const latestScores = latestNativeScores(); - const latestTargetDiagnostics = latestTargetDiagnosticsSummary(100); - - const historicalHeadline = policyengineHarness.isPolicyEngineComparison - ? { - baseline_label: baselineSlice.baselineLabel ?? null, - candidate_label: baselineSlice.candidateLabel ?? null, - calibration_target_profile: - baselineSlice.calibrationTargetProfile ?? null, - n_synthetic: comparisonMetadata.n_synthetic ?? null, - target_period: baselineSlice.targetPeriod ?? null, - baseline_composite_parity_loss: - policyengineHarness.baseline_composite_parity_loss ?? null, - candidate_composite_parity_loss: - policyengineHarness.candidate_composite_parity_loss ?? null, - composite_parity_loss_delta: - policyengineHarness.composite_parity_loss_delta ?? null, - baseline_mean_abs_relative_error: - policyengineHarness.baseline_mean_abs_relative_error ?? null, - candidate_mean_abs_relative_error: - policyengineHarness.candidate_mean_abs_relative_error ?? null, - mean_abs_relative_error_delta: - policyengineHarness.mean_abs_relative_error_delta ?? null, - slice_win_rate: policyengineHarness.slice_win_rate ?? null, - supported_target_rate: - policyengineHarness.supported_target_rate ?? null, - target_win_rate: allTargets.target_win_rate ?? null, - tag_summaries: tagSummaries, - } - : {}; - const headline = { - ...historicalHeadline, - ...latestHeadlinePatch(), - }; - - return NextResponse.json( - scrub({ - source_repo: "PolicyEngine/microplex-us", - source_artifacts: Object.entries(ARTIFACTS).map(([name, path]) => ({ - name, - path, - url: `${GITHUB_RAW}/${path}`, - })).concat([ - { - name: "latest_native_scores", - path: LATEST_MICROPLEX_NATIVE_SCORES_PATH, - url: "deployed-static-snapshot", - }, - { - name: "latest_target_diagnostics", - path: LATEST_MICROPLEX_TARGET_DIAGNOSTICS_PATH, - url: "deployed-static-snapshot", - }, - ]), - limitations: [ - "The headline native loss and target rows use the latest deployed static Microplex artifact snapshot.", - "The best/worst run history and IRS drilldown still come from older public summary JSONs committed in PolicyEngine/microplex-us.", - "Live microsim reform comparisons still require a hosted Python backend and configured Microplex H5 artifact root.", - ], - newer_runs: { - current_reader: "deployed_static_latest_artifact_snapshot", - public_branch: "PolicyEngine/microplex-us main", - run_bundle_manifest_key: RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - run_bundle_path_hint: RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - legacy_static_dashboard_path: LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH, - required_to_load_newer_runs: - "Update the deployed static artifact snapshot, point the dashboard at a generated Microplex artifact root, publish the run-bundle JSONs, or expose the run index/artifacts through an authenticated artifact service.", - not_loaded_reason: - "This Vercel deployment is reading the latest checked-in static artifact snapshot, not discovering private/generated run bundles at runtime.", - configured_run_discovery: configuredRuns, - }, - repo_structure: { - canonical_stage_count: 9, - current_commit_public_artifact_count: - Object.keys(ARTIFACTS).length + 2, - analysis_modes: [ - "microplex_vs_target_oracle", - "microplex_vs_us_data_comparator", - "run_to_run_microplex_comparison", - ], - generated_artifacts: GENERATED_ARTIFACT_CONTRACT, - full_target_diagnostics: { - available_in_committed_repo: true, - expected_path: RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - run_level_path: RUN_LEVEL_TARGET_DIAGNOSTICS_PATH, - manifest_key: RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - legacy_static_dashboard_path: LEGACY_STATIC_TARGET_DIAGNOSTICS_PATH, - static_dashboard_default_url: - LATEST_MICROPLEX_TARGET_DIAGNOSTICS_PATH, - producer_command: - `Run the Microplex PE-US-data rebuild/native audit pipeline; newer runs record manifest.artifacts.${RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY} = ${RUN_LEVEL_TARGET_DIAGNOSTICS_PATH}.`, - row_fields: TARGET_DIAGNOSTIC_ROW_FIELDS, - primary_use: - "Standalone Microplex aggregate-vs-target diagnostics; us-data baseline fields are optional comparator context.", - }, - run_index: { - path_hint: "run_index.duckdb", - query_helpers: [ - "list_us_microplex_target_delta_rows", - "compare_us_microplex_target_delta_rows", - "select_us_microplex_frontier_index_row", - ], - }, - }, - artifact_id: LATEST_MICROPLEX_ARTIFACT_ID, - verdict: parity.verdict ?? null, - headline, - native_scores: { - ...latestScores, - target_rows_available: targetRowsAvailable, - full_target_diagnostics_manifest_key: - RUN_LEVEL_TARGET_DIAGNOSTICS_MANIFEST_KEY, - }, - target_diagnostics: latestTargetDiagnostics, - regression_summary: { - total_scored_runs: regression.totalScoredRuns ?? null, - total_audited_runs: regression.totalAuditedRuns ?? null, - best_runs: Array.isArray(regression.bestRuns) - ? regression.bestRuns.slice(0, 10) - : [], - worst_runs: Array.isArray(regression.worstRuns) - ? regression.worstRuns.slice(0, 10) - : [], - largest_family_counts: regression.largestFamilyCounts ?? {}, - top3_family_counts: regression.top3FamilyCounts ?? {}, - target_counts_from_audits: regression.targetCountsFromAudits ?? {}, - }, - irs_drilldown: { - family: drilldown.family ?? null, - audits_where_family_leads: drilldown.auditsWhereFamilyLeads ?? null, - audits_with_matching_targets: - drilldown.auditsWithMatchingTargets ?? null, - lead_audits: Array.isArray(drilldown.leadAudits) - ? drilldown.leadAudits.slice(0, 10) - : [], - lead_target_counts: drilldown.leadTargetCounts ?? {}, - lead_filing_status_gap_summary: - drilldown.leadFilingStatusGapSummary ?? null, - lead_mfs_agi_gap_summary: drilldown.leadMFSAgiGapSummary ?? null, - }, - }), - ); - } catch (error) { - return NextResponse.json( - { detail: error instanceof Error ? error.message : String(error) }, - { status: 502 }, - ); - } -} diff --git a/frontend/app/api/microplex/target-diagnostics/route.ts b/frontend/app/api/microplex/target-diagnostics/route.ts deleted file mode 100644 index 200cf3c..0000000 --- a/frontend/app/api/microplex/target-diagnostics/route.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { NextResponse } from "next/server"; - -import { - latestTargetDiagnosticsPage, - scrub, -} from "@/lib/microplex/latest-artifact"; - -export const revalidate = 300; - -export async function GET(request: Request) { - return NextResponse.json(scrub(latestTargetDiagnosticsPage(request.url))); -} diff --git a/frontend/app/api/populace/compare/route.ts b/frontend/app/api/populace/compare/route.ts new file mode 100644 index 0000000..f281f14 --- /dev/null +++ b/frontend/app/api/populace/compare/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; + +import { loadComparison, loadPointerReleaseId, scrub } from "@/lib/populace/latest-artifact"; + +export const revalidate = 300; + +export async function GET(request: Request) { + const url = new URL(request.url); + let a = url.searchParams.get("a"); + let b = url.searchParams.get("b"); + try { + // Default b to the latest release; a is required for a real diff. + if (!b) b = (await loadPointerReleaseId(revalidate)).release_id; + if (!a || !b) { + return NextResponse.json( + { detail: "Provide two releases to compare via ?a=&b=." }, + { status: 400 }, + ); + } + const comparison = await loadComparison(a, b, revalidate); + return NextResponse.json(scrub(comparison)); + } catch (error) { + return NextResponse.json( + { detail: error instanceof Error ? error.message : String(error) }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/api/populace/releases/route.ts b/frontend/app/api/populace/releases/route.ts new file mode 100644 index 0000000..3873920 --- /dev/null +++ b/frontend/app/api/populace/releases/route.ts @@ -0,0 +1,28 @@ +import { NextResponse } from "next/server"; + +import { loadPointerReleaseId, loadReleases, scrub } from "@/lib/populace/latest-artifact"; + +export const revalidate = 300; + +export async function GET() { + try { + const [releases, pointer] = await Promise.all([ + loadReleases(revalidate), + loadPointerReleaseId(revalidate).catch(() => ({ release_id: "", updated_at: null })), + ]); + return NextResponse.json( + scrub({ + latest_release_id: pointer.release_id, + updated_at: pointer.updated_at, + // Releases that carry the per-target diagnostics (compare-able). + releases: releases.filter((r) => r.has_calibration), + all_releases: releases, + }), + ); + } catch (error) { + return NextResponse.json( + { detail: error instanceof Error ? error.message : String(error) }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/api/populace/route.ts b/frontend/app/api/populace/route.ts new file mode 100644 index 0000000..17e2e6f --- /dev/null +++ b/frontend/app/api/populace/route.ts @@ -0,0 +1,54 @@ +import { NextResponse } from "next/server"; + +import { + POPULACE_HF_REPO, + POPULACE_HF_REVISION, + asObject, + hfResolveUrl, + latestPopulaceCalibrationHighlights, + latestPopulaceCalibrationSummary, + loadRelease, + scrub, +} from "@/lib/populace/latest-artifact"; + +export const revalidate = 300; + +export async function GET(request: Request) { + const release = new URL(request.url).searchParams.get("release") ?? "latest"; + try { + const cal = await loadRelease(release, revalidate); + const calibration = latestPopulaceCalibrationSummary(cal); + const highlights = latestPopulaceCalibrationHighlights(cal, 15); + const prefix = `releases/${cal.release_id}`; + return NextResponse.json( + scrub({ + source_repo: POPULACE_HF_REPO, + repo_type: "dataset", + revision: POPULACE_HF_REVISION, + source: "huggingface_live", + release_id: cal.release_id, + updated_at: cal.updated_at, + source_artifacts: [ + { name: "latest_pointer", path: "latest.json", url: hfResolveUrl("latest.json") }, + { name: "build_manifest", path: `${prefix}/build_manifest.json`, url: hfResolveUrl(`${prefix}/build_manifest.json`) }, + { name: "release_manifest", path: `${prefix}/release_manifest.json`, url: hfResolveUrl(`${prefix}/release_manifest.json`) }, + { name: "calibration_diagnostics", path: `${prefix}/calibration_diagnostics.json`, url: hfResolveUrl(`${prefix}/calibration_diagnostics.json`) }, + ], + limitations: [ + "Everything on this page is read live from the policyengine/populace-us Hugging Face dataset; the current release is resolved through latest.json.", + "Loss values are the calibrator's own metric for this release; their scale is not comparable across releases that calibrate to different target surfaces.", + ], + build_manifest: cal.build_manifest, + release_manifest: cal.release_manifest, + gates: asObject(cal.build_manifest.gates), + calibration, + highlights, + }), + ); + } catch (error) { + return NextResponse.json( + { detail: error instanceof Error ? error.message : String(error) }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/api/populace/target-diagnostics/route.ts b/frontend/app/api/populace/target-diagnostics/route.ts new file mode 100644 index 0000000..2edf509 --- /dev/null +++ b/frontend/app/api/populace/target-diagnostics/route.ts @@ -0,0 +1,22 @@ +import { NextResponse } from "next/server"; + +import { + latestPopulaceTargetDiagnosticsPage, + loadRelease, + scrub, +} from "@/lib/populace/latest-artifact"; + +export const revalidate = 300; + +export async function GET(request: Request) { + const release = new URL(request.url).searchParams.get("release") ?? "latest"; + try { + const cal = await loadRelease(release, revalidate); + return NextResponse.json(scrub(latestPopulaceTargetDiagnosticsPage(request.url, cal))); + } catch (error) { + return NextResponse.json( + { detail: error instanceof Error ? error.message : String(error) }, + { status: 502 }, + ); + } +} diff --git a/frontend/app/budget-impact/page.tsx b/frontend/app/budget-impact/page.tsx deleted file mode 100644 index 7b4f28e..0000000 --- a/frontend/app/budget-impact/page.tsx +++ /dev/null @@ -1 +0,0 @@ -export { default } from "@/components/microplex/reform-benchmarks-page"; diff --git a/frontend/app/comparison/page.tsx b/frontend/app/comparison/page.tsx deleted file mode 100644 index 4403904..0000000 --- a/frontend/app/comparison/page.tsx +++ /dev/null @@ -1,600 +0,0 @@ -"use client"; - -import Link from "next/link"; -import { - Badge, - Card, - CardContent, - CardHeader, - CardTitle, - Skeleton, - Stack, - Text, - Title, - formatNumber, -} from "@policyengine/ui-kit"; - -import { AppShell } from "@/components/layout/app-shell"; -import { useMicroplex } from "@/lib/api/hooks/use-microplex"; -import { useSummary } from "@/lib/api/hooks/use-summary"; -import { useTargetInventorySummary } from "@/lib/api/hooks/use-target-inventory"; -import { useRunContext } from "@/lib/run-context"; - -function pct(value: number | null | undefined, digits = 1): string { - if (value === null || value === undefined || !Number.isFinite(value)) { - return "—"; - } - return `${(value * 100).toFixed(digits)}%`; -} - -function num(value: number | null | undefined): string { - if (value === null || value === undefined || !Number.isFinite(value)) { - return "—"; - } - return formatNumber(value); -} - -function Kpi({ - label, - value, - hint, -}: { - label: string; - value: string; - hint?: string; -}) { - return ( -
- - {label} - -
{value}
- {hint && ( - - {hint} - - )} -
- ); -} - -function DeltaBadge({ - value, - improveIsLower = true, -}: { - value: number | null | undefined; - improveIsLower?: boolean; -}) { - if (value === null || value === undefined || !Number.isFinite(value)) { - return —; - } - const improved = improveIsLower ? value < 0 : value > 0; - const variant: "success" | "error" | "secondary" = - Math.abs(value) < 1e-9 ? "secondary" : improved ? "success" : "error"; - const sign = value > 0 ? "+" : ""; - return ( - - {sign} - {num(value)} - - ); -} - -export default function ComparisonPage() { - const { dataset, run } = useRunContext(); - const summary = useSummary(); - const microplex = useMicroplex(); - const inventory = useTargetInventorySummary(); - const microplexTargetRefs = microplex.data - ? [ - ...microplex.data.regression_summary.target_counts_from_audits.map( - (target) => ({ ...target, source: "regression audits" }), - ), - ...microplex.data.irs_drilldown.lead_target_counts.map((target) => ({ - ...target, - source: "IRS drilldown", - })), - ].slice(0, 12) - : []; - - return ( - - -
- Comparison - - Side-by-side orientation for the selected us-data run and the - public Microplex parity artifact. - -
- -
- - -
- us-data run - selected dashboard run -
- - - - - Dataset {dataset ?? "not selected"} · Run{" "} - {run ?? "not selected"} - - {summary.isLoading && } - {summary.error && ( - - Failed to load us-data summary: {String(summary.error)} - - )} - {summary.data && ( -
- - - - -
- )} -
- - Open summary - - - Inspect targets - - - Trace variables - -
- - - - - - -
- Microplex artifact - public parity JSON -
- - - - {microplex.isLoading && } - {microplex.error && ( - - Failed to load Microplex artifact: {String(microplex.error)} - - )} - {microplex.data && ( - <> - - Artifact{" "} - - {microplex.data.artifact_id} - {" "} - · profile{" "} - - {microplex.data.headline.calibration_target_profile} - - -
- - - - -
- - )} -
- - Open Microplex overview - - - Inspect Microplex pipeline - -
- - - -
- - - - Interpretation boundary - - - - The Microplex side currently comes from public aggregate JSON - artifacts. The us-data side comes from the loaded dashboard run. - A full apples-to-apples target-by-target comparison needs the - generated Microplex H5 or full PE-native target diagnostic JSON - to be published and loaded into this app. - - - - - {microplex.data?.repo_structure && ( - - - Microplex artifact structure - - - - - The Microplex repo has a richer generated artifact structure - than the committed public JSONs. Newer run bundles record - the full target diagnostics manifest artifact{" "} - - { - microplex.data.repo_structure.full_target_diagnostics - .manifest_key - } - - , pointing to{" "} - - { - microplex.data.repo_structure.full_target_diagnostics - .run_level_path - } - - . - - -
- - - -
- -
- - - - - - - - - - - {microplex.data.repo_structure.generated_artifacts.map( - (artifact) => ( - - - - - - - ), - )} - -
ArtifactPath hintProducerPublic
-
- {artifact.name} -
- - {artifact.description} - - 		- {artifact.path_hint} - - {artifact.producer} - - {artifact.public_committed ? "yes" : "no"} -
-
- - - - )} - - - - Target comparison surface - - - - - Yes, this repo has the target definitions. What we do not yet - have in public form is Microplex's full per-target estimate - table, so this section compares target inventory and public - Microplex target references rather than target-by-target model - errors. - - -
- - - -
- -
-
- - - - - - - - - - - {inventory.data?.tiers.slice(0, 8).map((tier) => ( - - - - - - - ))} - {!inventory.data && ( - - - - )} - -
Inventory tierRecordsMatched to DBMatch rate
- {tier.tier} - - {num(tier.total_records)} - - {num(tier.matched_to_db)} - - {pct(tier.match_rate)} -
- Loading target inventory... -
-
- -
- - - - - - - - - - - {microplexTargetRefs.map((target) => ( - - - - - - - ))} - {microplexTargetRefs.length === 0 && ( - - - - )} - -
Microplex target referenceSourceCountMean Δ
- {target.target} - {target.source} - {num(target.count)} - - {num(target.weightedTermDeltaMean)} -
- Loading Microplex target references... -
-
-
- -
- - Open full target inventory - - - Open us-data target estimates - -
- - - - - {microplex.data?.native_scores && ( - - -
- Microplex aggregates vs PE targets - - {microplex.data.native_scores.candidate_beats_baseline - ? "candidate beats baseline" - : "baseline lower native loss"} - -
- - - - - These are the native PE aggregate-vs-target summary metrics - from the public Microplex parity artifact. Lower loss is - better. The row-level table is generated in newer run bundles - as{" "} - - { - microplex.data.native_scores - .full_target_diagnostics_path - } - - , but those generated bundles are not committed publicly. - - -
- -
- - Native loss delta - -
- -
- - candidate minus baseline - -
- - - - -
- - - - )} - - - ); -} diff --git a/frontend/app/inventory/page.tsx b/frontend/app/inventory/page.tsx deleted file mode 100644 index 387812f..0000000 --- a/frontend/app/inventory/page.tsx +++ /dev/null @@ -1,324 +0,0 @@ -"use client"; - -import { - Card, - CardHeader, - CardTitle, - CardContent, - Badge, - Skeleton, - Stack, - Title, - Text, - Input, - formatNumber, -} from "@policyengine/ui-kit"; -import { AppShell } from "@/components/layout/app-shell"; -import { DataTable } from "@/components/shared/InteractiveDataTable"; -import { - useTargetInventory, - useTargetInventorySummary, - type InventoryRow, -} from "@/lib/api/hooks/use-target-inventory"; -import { useState } from "react"; - -const TIER_OPTIONS = ["all", "db", "csv", "python", "generator", "yaml"] as const; -type TierOption = (typeof TIER_OPTIONS)[number]; - -const IN_DB_OPTIONS = ["any", "in-db", "out-of-db"] as const; -type InDbOption = (typeof IN_DB_OPTIONS)[number]; - -const PAGE_SIZE = 50; - -function TierBadge({ tier }: { tier: InventoryRow["storage_tier"] }) { - const styles: Record = { - db: "bg-slate-200 text-slate-800 border-slate-300", - csv: "bg-amber-100 text-amber-800 border-amber-300", - python: "bg-blue-100 text-blue-800 border-blue-300", - generator: "bg-purple-100 text-purple-800 border-purple-300", - yaml: "bg-emerald-100 text-emerald-800 border-emerald-300", - }; - return ( - - {tier} - - ); -} - -function InventoryTable() { - const [tier, setTier] = useState("all"); - const [inDb, setInDb] = useState("any"); - const [search, setSearch] = useState(""); - const [page, setPage] = useState(0); - - const inventory = useTargetInventory({ - tier: tier === "all" ? undefined : tier, - in_db: - inDb === "in-db" ? true : inDb === "out-of-db" ? false : undefined, - search: search || undefined, - limit: PAGE_SIZE, - offset: page * PAGE_SIZE, - }); - - const lastPage = inventory.data - ? Math.max(0, Math.ceil(inventory.data.total / PAGE_SIZE) - 1) - : 0; - - const columns = [ - { - key: "storage_tier", - header: "Tier", - format: (val: unknown) => , - }, - { - key: "in_db", - header: "In DB?", - format: (val: unknown) => - val ? ( - in DB - ) : ( - authored only - ), - }, - { key: "variable", header: "Variable" }, - { - key: "period", - header: "Period", - align: "right" as const, - format: (val: unknown) => (val == null ? "—" : String(val)), - }, - { - key: "geo_level", - header: "Geo", - format: (val: unknown, row: Record) => { - const gid = row.geographic_id; - const lvl = String(val ?? "—"); - return gid ? `${lvl} / ${gid}` : lvl; - }, - }, - { - key: "value", - header: "Target", - align: "right" as const, - format: (val: unknown) => - val == null ? — : formatNumber(Number(val)), - }, - { - key: "estimate", - header: "PE aggregate", - align: "right" as const, - format: (val: unknown) => - val == null ? — : formatNumber(Number(val)), - }, - { - key: "rel_error", - header: "Rel. err", - align: "right" as const, - format: (val: unknown) => { - if (val == null) return —; - const v = Number(val); - const abs = Math.abs(v); - const variant = - abs > 0.5 - ? "error" - : abs > 0.2 - ? "warning" - : abs > 0.05 - ? "secondary" - : "success"; - const display = - abs >= 1 ? `${(v * 100).toFixed(0)}%` : `${(v * 100).toFixed(1)}%`; - return {display}; - }, - }, - { - key: "source_path", - header: "Source", - format: (val: unknown) => ( - - {String(val).replace("storage/calibration_targets/", "…/")} - - ), - }, - { - key: "constraints", - header: "Constraints", - format: (val: unknown) => { - const cs = val as [string, string, string][]; - if (!cs || cs.length === 0) - return —; - return ( -
- {cs.map(([v, op, value], i) => ( - - {v} {op} {value} - - ))} -
- ); - }, - }, - ]; - - const btn = - "h-8 min-w-8 rounded border border-border bg-background px-2 text-sm hover:bg-muted disabled:opacity-40 disabled:cursor-not-allowed"; - - return ( - -
-
- { - setSearch(e.target.value); - setPage(0); - }} - /> -
-
- Tier - -
-
- DB - -
-
- - {inventory.data ? ( - <> - - {formatNumber(inventory.data.total)} matching records · page{" "} - {page + 1} of {lastPage + 1} - - -
- - - - {page + 1} / {lastPage + 1} - - - -
- - ) : ( - - )} - - ); -} - -function CoverageSummary() { - const summary = useTargetInventorySummary(); - if (!summary.data) return ; - const tot = summary.data.tiers.reduce((s, t) => s + t.total_records, 0); - const matched = summary.data.tiers.reduce((s, t) => s + t.matched_to_db, 0); - - return ( - - - Cross-tier audit - - - - - {formatNumber(tot)}{" "} - authored records across {summary.data.tiers.length} non-DB tiers;{" "} - {formatNumber(matched)}{" "} - ({((matched / tot) * 100).toFixed(1)}%) match a target in the - loaded policy_data.db ({formatNumber(summary.data.db_total)} rows). - - - - - - - - - - - - {summary.data.tiers.map((t) => { - const rate = (t.match_rate ?? 0) * 100; - return ( - - - - - - - ); - })} - -
SourceRecordsMatched%
{t.tier}{formatNumber(t.total_records)}{formatNumber(t.matched_to_db)} - {rate.toFixed(1)}% -
- {summary.data.parsers_missing.length > 0 && ( - - No parser yet for: {summary.data.parsers_missing.join(", ")} - - )} - - - - ); -} - -export default function TargetInventoryPage() { - return ( - - -
- Target inventory - - Every target authored across all 5 storage tiers in{" "} - policyengine_us_data: CSV source files, Python - constants, generators, the YAML config, and the compiled{" "} - policy_data.db. The in-DB badge - tells you whether each authored row made it into the - currently-loaded calibration. - -
- - - - - - Browse records - - - - - - - - ); -} diff --git a/frontend/app/microplex/diagnostics/page.tsx b/frontend/app/microplex/diagnostics/page.tsx deleted file mode 100644 index bee8ff3..0000000 --- a/frontend/app/microplex/diagnostics/page.tsx +++ /dev/null @@ -1 +0,0 @@ -export { default } from "../page"; diff --git a/frontend/app/microplex/page.tsx b/frontend/app/microplex/page.tsx deleted file mode 100644 index 80e5758..0000000 --- a/frontend/app/microplex/page.tsx +++ /dev/null @@ -1,1564 +0,0 @@ -"use client"; - -import { useEffect, useState, type ReactNode } from "react"; - -import { - Badge, - Card, - CardContent, - CardHeader, - CardTitle, - Stack, - Text, - Title, - formatNumber, -} from "@policyengine/ui-kit"; - -import { AppShell } from "@/components/layout/app-shell"; -import { - DataTable, - type SortState, -} from "@/components/shared/InteractiveDataTable"; -import { LoadingBlock } from "@/components/shared/LoadingBlock"; -import { - useMicroplex, - useMicroplexReformComparison, - useMicroplexTargetDiagnostics, -} from "@/lib/api/hooks/use-microplex"; -import { usePathname } from "next/navigation"; - -const TARGET_DIAGNOSTICS_LIMIT = 100; - -const TARGET_FAMILY_OPTIONS = [ - "national_census_other", - "national_infants", - "national_irs_other", - "national_net_worth", - "national_population_by_age", - "national_spm_threshold_agi", - "national_spm_threshold_count", - "national_ssa", - "national_tax_expenditures", - "other", - "state_aca_enrollment", - "state_aca_spending", - "state_age_distribution", - "state_agi_distribution", - "state_population", - "state_population_under_5", - "state_real_estate_taxes", - "state_snap_cost", - "state_snap_households", -]; - -const STATE_OPTIONS = [ - "AL", - "AK", - "AZ", - "AR", - "CA", - "CO", - "CT", - "DE", - "DC", - "FL", - "GA", - "HI", - "ID", - "IL", - "IN", - "IA", - "KS", - "KY", - "LA", - "ME", - "MD", - "MA", - "MI", - "MN", - "MS", - "MO", - "MT", - "NE", - "NV", - "NH", - "NJ", - "NM", - "NY", - "NC", - "ND", - "OH", - "OK", - "OR", - "PA", - "RI", - "SC", - "SD", - "TN", - "TX", - "UT", - "VT", - "VA", - "WA", - "WV", - "WI", - "WY", -]; - -function fmt(v: number | null | undefined, opts: { pct?: boolean } = {}) { - if (v == null || !Number.isFinite(v)) return "—"; - if (opts.pct) return `${(v * 100).toFixed(1)}%`; - if (Math.abs(v) >= 1000 || Number.isInteger(v)) return formatNumber(v); - return v.toFixed(4); -} - -function deltaBadge(v: number | null | undefined, improveIsLower = true) { - if (v == null || !Number.isFinite(v)) { - return —; - } - const improved = improveIsLower ? v < 0 : v > 0; - const variant: "success" | "error" | "secondary" = - Math.abs(v) < 1e-9 ? "secondary" : improved ? "success" : "error"; - const sign = v > 0 ? "+" : ""; - return ( - - {sign} - {fmt(v)} - - ); -} - -function signedFmt(v: number | null | undefined) { - if (v == null || !Number.isFinite(v)) return "—"; - const sign = v > 0 ? "+" : ""; - return ( - - {sign} - {fmt(v)} - - ); -} - -function signedPct(v: number | null | undefined) { - if (v == null || !Number.isFinite(v)) return "—"; - const sign = v > 0 ? "+" : ""; - return ( - - {sign} - {(v * 100).toFixed(2)}% - - ); -} - -const bestWorstColumns = [ - { key: "artifactPath", header: "Run", format: (v: unknown) => ( - - {String(v)} - - ) }, - { - key: "lossDelta", - header: ( - - Loss delta - - ), - align: "right" as const, - format: (v: unknown) => deltaBadge(Number(v)), - }, - { - key: "candidateBeatsBaseline", - header: "Beats us-data?", - format: (v: unknown) => - v ? yes : no, - }, - { - key: "largestRegressingFamily", - header: "Worst family", - format: (v: unknown) => ( - {v == null ? "—" : String(v)} - ), - }, - { - key: "largestRegressingFamilyDelta", - header: ( - - Family delta - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : Number(v).toFixed(3), - }, -]; - -const familyColumns = [ - { key: "family", header: "Family", format: (v: unknown) => ( - {String(v)} - ) }, - { - key: "rank1Count", - header: "#1", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "rank2Count", - header: "#2", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "rank3Count", - header: "#3", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "top3Count", - header: ( - - Top-3 total - - ), - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, -]; - -const leadColumns = [ - { - key: "target", - header: "Target", - format: (v: unknown) => ( - {String(v)} - ), - }, - { - key: "weightedTermDelta", - header: ( - - Weighted delta - - ), - align: "right" as const, - format: (v: unknown) => Number(v).toFixed(2), - }, -]; - -function metricTone(value: number | null | undefined, improveIsLower = true) { - if (value == null || !Number.isFinite(value) || Math.abs(value) < 1e-9) { - return "secondary" as const; - } - return (improveIsLower ? value < 0 : value > 0) ? "success" as const : "error" as const; -} - -function MetricTile({ - label, - value, - detail, - badge, -}: { - label: ReactNode; - value: string; - detail?: string; - badge?: ReactNode; -}) { - return ( -
-
- - {label} - - {badge} -
-
{value}
- {detail && ( - - {detail} - - )} -
- ); -} - -function SectionIntro({ - title, - children, -}: { - title: string; - children?: ReactNode; -}) { - return ( -
- {title} - {children && ( - - {children} - - )} -
- ); -} - -function HelpText({ children, title }: { children: ReactNode; title: string }) { - return ( - - {children} - - ? - - - {title} - - - ); -} - -function CompactSearchInput({ - label, - value, - onChange, - placeholder, -}: { - label: string; - value: string; - onChange: (value: string) => void; - placeholder: string; -}) { - return ( - - ); -} - -function CompactSelect({ - label, - value, - options, - onChange, - disabled, - className = "", -}: { - label: string; - value: string; - options: { value: string; label: string }[]; - onChange: (value: string) => void; - disabled?: boolean; - className?: string; -}) { - return ( - - ); -} - -const targetCountColumns = [ - { - key: "target", - header: "Target", - format: (v: unknown) => ( - {String(v)} - ), - }, - { - key: "count", - header: "Count", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "weightedTermDeltaMean", - header: ( - - Mean weighted delta - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : Number(v).toFixed(2), - }, - { - key: "weightedTermDeltaSum", - header: ( - - Total weighted delta - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : Number(v).toFixed(2), - }, -]; - -const filingStatusGapColumns = [ - { - key: "filingStatus", - header: "Filing status", - format: (v: unknown) => ( - {String(v)} - ), - }, - { - key: "meanAbsWeightedCountDelta", - header: ( - - Mean count gap - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : formatNumber(Number(v)), - }, - { - key: "weightedCountDeltaSum", - header: ( - - Total count gap - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : formatNumber(Number(v)), - }, - { - key: "positiveCount", - header: "+ audits", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "negativeCount", - header: "- audits", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, -]; - -const agiGapColumns = [ - { - key: "agiBin", - header: "AGI bin", - format: (v: unknown) => ( - {String(v)} - ), - }, - { - key: "meanAbsWeightedCountDelta", - header: ( - - Mean count gap - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : formatNumber(Number(v)), - }, - { - key: "weightedCountDeltaSum", - header: ( - - Total count gap - - ), - align: "right" as const, - format: (v: unknown) => - v == null ? "—" : formatNumber(Number(v)), - }, - { - key: "positiveCount", - header: "+ audits", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "negativeCount", - header: "- audits", - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, -]; - -function targetDiagnosticsColumns(compareWithUsData: boolean) { - return [ - { - key: "target_id", - header: "Target", - format: (v: unknown, row: Record) => ( - - {String(row.target_name ?? v ?? "—")} - {row.target_name != null && v != null && String(row.target_name) !== String(v) ? ( - - ID {String(v)} - - ) : null} - - ), - }, - { - key: "family", - header: ( - - Family - - ), - format: (v: unknown, row: Record) => ( - - {String(v ?? row.target_family ?? "—")} - - ), - }, - { - key: "target_value", - header: ( - - Target value - - ), - align: "right" as const, - format: (v: unknown) => (v == null ? "—" : fmt(Number(v))), - }, - { - key: "us_data_aggregate", - header: ( - - us-data aggregate - - ), - align: "right" as const, - format: (v: unknown, row: Record) => { - const value = v ?? row.from_estimate; - return value == null ? "—" : fmt(Number(value)); - }, - }, - { - key: "microplex_aggregate", - header: ( - - Microplex aggregate - - ), - align: "right" as const, - format: (v: unknown, row: Record) => { - const value = v ?? row.to_estimate; - return value == null ? "—" : fmt(Number(value)); - }, - }, - { - key: "microplex_vs_target_relative", - header: ( - - Microplex vs target - - ), - align: "right" as const, - format: (v: unknown, row: Record) => { - let value = v; - if (value == null) { - const target = Number(row.target_value); - const microplex = Number(row.to_estimate ?? row.microplex_aggregate); - if (Number.isFinite(target) && target !== 0 && Number.isFinite(microplex)) { - value = (microplex - target) / Math.abs(target); - } - } - return signedPct(value == null ? null : Number(value)); - }, - }, - { - key: "microplex_vs_us_data_relative", - header: ( - - Microplex vs us-data - - ), - align: "right" as const, - format: (v: unknown, row: Record) => { - let value = v; - if (value == null) { - const usData = Number(row.from_estimate ?? row.us_data_aggregate); - const microplex = Number(row.to_estimate ?? row.microplex_aggregate); - if (Number.isFinite(usData) && usData !== 0 && Number.isFinite(microplex)) { - value = (microplex - usData) / Math.abs(usData); - } - } - return signedPct(value == null ? null : Number(value)); - }, - }, - { - key: "closer_dataset", - header: ( - - Closer - - ), - format: (v: unknown) => - v === "microplex" ? ( - Microplex - ) : v === "us-data" ? ( - us-data - ) : v === "tie" ? ( - tie - ) : ( - "—" - ), - }, - { - key: "delta_absolute_error", - header: ( - - Abs error delta - - ), - align: "right" as const, - format: (v: unknown, row: Record) => { - let value = v; - if (value == null) { - const target = Number(row.target_value); - const fromEstimate = Number(row.from_estimate ?? row.us_data_aggregate); - const toEstimate = Number(row.to_estimate ?? row.microplex_aggregate); - if ( - Number.isFinite(target) && - Number.isFinite(fromEstimate) && - Number.isFinite(toEstimate) - ) { - value = Math.abs(toEstimate - target) - Math.abs(fromEstimate - target); - } - } - return value == null ? "—" : deltaBadge(Number(value)); - }, - }, - { - key: "supported_by_microplex", - header: ( - - Supported - - ), - format: (v: unknown) => - v === true ? ( - yes - ) : v === false ? ( - no - ) : ( - "—" - ), - }, - ].filter( - (column) => - compareWithUsData || - ![ - "us_data_aggregate", - "microplex_vs_us_data_relative", - "closer_dataset", - "delta_absolute_error", - ].includes(column.key), - ); -} - -function ReformComparisonCard({ - comparison, - isLoading, - error, - reformId, - onReformChange, -}: { - comparison: ReturnType["data"]; - isLoading: boolean; - error: unknown; - reformId: string; - onReformChange: (value: string) => void; -}) { - const outcome = comparison?.outcomes?.[0]; - const reformOptions = comparison?.available_reforms ?? [ - { - id: "american_family_act_2025", - label: "American Family Act 2025 CTC expansion", - }, - { - id: "working_parents_tax_relief_act_2026", - label: "Working Parents Tax Relief Act EITC enhancement", - }, - { - id: "halve_joint_eitc_phase_out_rate", - label: "Halve joint-filer EITC phase-out rate", - }, - ]; - return ( - - -
- Microsim reform comparison - {comparison?.available ? ( - ran locally - ) : ( - - {isLoading ? "running" : "not available"} - - )} -
- - - - - - Runs the same PolicyEngine-US reform over the incumbent us-data H5 - and the configured Microplex H5. This is a reform-sensitivity check: - it tells us whether the candidate dataset produces a comparable - aggregate policy impact, not whether the reform itself is calibrated. - - - {isLoading && ( - - )} - {error ? ( - - Failed to run microsim comparison: {String(error)} - - ) : null} - {!isLoading && comparison && !comparison.available && ( - - {comparison.reason ?? "No Microplex H5 is configured."} - - )} - {comparison?.available && outcome && ( - <> -
- - us-data {outcome.variable} delta - - } - value={fmt(outcome.us_data.delta)} - detail={`baseline ${fmt(outcome.us_data.baseline.total)}`} - /> - - Microplex {outcome.variable} delta - - } - value={fmt(outcome.microplex.delta)} - detail={`baseline ${fmt(outcome.microplex.baseline.total)}`} - /> - - Delta ratio - - } - value={fmt(outcome.microplex_delta_as_share_of_us_data)} - detail={`gap ${fmt(outcome.delta_gap)}`} - /> -
- - Reform: {comparison.reform?.label ?? "unknown"} for period{" "} - {comparison.period}. Microplex artifact{" "} - - {comparison.microplex_bundle?.artifact_id ?? "unknown"} - - . us-data records: {fmt(outcome.us_data.baseline.record_count)}; - Microplex records: {fmt(outcome.microplex.baseline.record_count)} - {comparison.runtime_seconds != null - ? `; runtime ${comparison.runtime_seconds.toFixed(1)}s` - : ""} - . - {comparison.reform?.source_url ? ( - <> - {" "} - - Source - - . - - ) : null} - - - )} - - - - ); -} - -export default function MicroplexPage() { - const pathname = usePathname(); - const isDiagnosticsView = pathname.startsWith("/microplex/diagnostics"); - const { data, isLoading, error } = useMicroplex(); - const [selectedReformId, setSelectedReformId] = useState( - "american_family_act_2025", - ); - const [compareWithUsData, setCompareWithUsData] = useState(true); - const [targetSearch, setTargetSearch] = useState(""); - const [targetFamily, setTargetFamily] = useState(""); - const [targetGeoLevel, setTargetGeoLevel] = useState(""); - const [targetDirection, setTargetDirection] = useState(""); - const [targetState, setTargetState] = useState(""); - const [targetSupported, setTargetSupported] = useState(""); - const [targetInLoss, setTargetInLoss] = useState(""); - const [targetOffset, setTargetOffset] = useState(0); - const [targetSort, setTargetSort] = useState({ - key: "microplex_vs_target_relative", - direction: "desc", - }); - const reformComparison = useMicroplexReformComparison(selectedReformId); - const targetDiagnosticsQuery = useMicroplexTargetDiagnostics({ - limit: TARGET_DIAGNOSTICS_LIMIT, - offset: targetOffset, - search: targetSearch || undefined, - family: targetFamily || undefined, - geo_level: targetGeoLevel || undefined, - microplex_target_direction: targetDirection || undefined, - state: targetState || undefined, - supported: targetSupported || undefined, - in_loss: targetInLoss || undefined, - sort_by: targetSort?.key, - sort_dir: targetSort?.direction, - }); - - useEffect(() => { - setTargetOffset(0); - }, [ - targetSearch, - targetFamily, - targetGeoLevel, - targetDirection, - targetState, - targetSupported, - targetInLoss, - targetSort, - ]); - - useEffect(() => { - if (targetGeoLevel === "national" && targetState) { - setTargetState(""); - } - }, [targetGeoLevel, targetState]); - - if (isLoading) - return ( - - - - ); - if (error) - return ( - - - Failed to load microplex artifacts: {String(error)} - - - ); - if (!data) return null; - - const h = data.headline; - const native = data.native_scores; - const hasCurrentArtifactScore = - native.source === "configured_run_bundle" || - native.source === "deployed_static_artifact"; - const verdictEntries = data.verdict ? Object.entries(data.verdict) : []; - const leadAuditTargets = data.irs_drilldown.lead_audits.flatMap((a) => - (a.matchingTargets ?? []).map((t) => ({ ...t, audit: a.artifactPath })), - ); - const topFamilies = data.regression_summary.top3_family_counts.slice(0, 5); - const topTargets = data.regression_summary.target_counts_from_audits.slice(0, 8); - const targetDiagnostics = - targetDiagnosticsQuery.data ?? data.target_diagnostics; - const targetDiagnosticsLoading = - targetDiagnosticsQuery.isLoading || targetDiagnosticsQuery.isFetching; - const filteredTargetTotal = targetDiagnostics.total_targets ?? 0; - const unfilteredTargetTotal = - targetDiagnostics.unfiltered_total_targets ?? data.target_diagnostics.total_targets; - - return ( - - -
- Microplex target performance - - Read-only aggregate view of the latest deployed Microplex artifact - snapshot, with historical run summaries from{" "} - - PolicyEngine/microplex-us - - . The primary question here is how Microplex scores against the{" "} - - PolicyEngine target oracle - - . The incumbent us-data baseline is shown only as comparison - context. - -
- - {!isDiagnosticsView && ( - <> - - - - us-data baseline means the incumbent - PolicyEngine us-data dataset scored through the same target - oracle. It is useful context, but the main question is still: - how close is Microplex to each target value? - - - - - - - - -
- - {hasCurrentArtifactScore - ? "Latest artifact broad target score" - : "Historical broad target score"} - - - {native.candidate_beats_baseline - ? "beats us-data baseline" - : "us-data baseline lower loss"} - -
- - - -
- - {hasCurrentArtifactScore - ? "Microplex native loss" - : "Historical Microplex native loss"} - - } - value={fmt(native.candidate_enhanced_cps_native_loss)} - detail={`us-data baseline ${fmt( - native.baseline_enhanced_cps_native_loss, - )}`} - badge={deltaBadge(native.enhanced_cps_native_loss_delta)} - /> - - Microplex unweighted MSRE - - } - value={fmt(native.candidate_unweighted_msre)} - detail={`us-data baseline ${fmt(native.baseline_unweighted_msre)}`} - badge={deltaBadge(native.unweighted_msre_delta)} - /> - - Targets scored - - } - value={fmt(native.n_targets_kept)} - detail={`${fmt(native.n_national_targets)} national, ${fmt( - native.n_state_targets, - )} state, out of ${fmt(native.n_targets_total)} total`} - /> -
- - Lower loss is better.{" "} - {hasCurrentArtifactScore ? ( - <> - These values come from latest deployed artifact{" "} - - {native.artifact_id ?? data.artifact_id ?? "unknown"} - - {native.source_path ? ( - <> - {" "}via{" "} - - {native.source_path} - - - ) : null} - . - - ) : ( - <> - These values come from the committed public parity artifact{" "} - - {data.artifact_id ?? "unknown"} - - , which does not include candidate or baseline dataset paths - and should not be read as a current live Microplex score. - - )}{" "} - The full row-level aggregate table is read from{" "} - - {native.full_target_diagnostics_path} - - {" "}under manifest key{" "} - - {native.full_target_diagnostics_manifest_key} - - . - - - - - - )} - - {isDiagnosticsView && ( - - -
-
- Target diagnostics rows - - Microplex aggregate fit against each calibration target. - -
-
- - - {targetDiagnostics.available ? "loaded" : "not loaded"} - -
-
- - - - - Row-level target diagnostics from{" "} - - {targetDiagnostics.path ?? native.full_target_diagnostics_path} - - . Showing {fmt(targetDiagnostics.targets.length)} of{" "} - {fmt(filteredTargetTotal)} matching rows - {filteredTargetTotal !== unfilteredTargetTotal - ? ` from ${fmt(unfilteredTargetTotal)} total` - : ""} - . - -
-
- - ({ - value: family, - label: family, - })), - ]} - className="w-[250px]" - /> - - - ({ - value: state, - label: state, - })), - ]} - className="w-[125px]" - /> - - - -
-
-
- - {targetDiagnosticsLoading - ? "Loading filtered rows..." - : `Rows ${fmt(targetOffset + 1)}-${fmt( - targetOffset + targetDiagnostics.targets.length, - )} of ${fmt(filteredTargetTotal)}`} - -
- - -
-
- [] - } - sortable - sort={targetSort} - onSortChange={(sort) => { - const nextSort = - sort ?? - (targetSort - ? { - key: targetSort.key, - direction: - targetSort.direction === "desc" ? "asc" : "desc", - } - : null); - setTargetSort(nextSort); - setTargetOffset(0); - }} - styles={{ - table: { minWidth: compareWithUsData ? 1680 : 1080 }, - header: { - paddingLeft: 10, - paddingRight: 10, - whiteSpace: "nowrap", - }, - cell: { - paddingLeft: 10, - paddingRight: 10, - verticalAlign: "top", - whiteSpace: "nowrap", - }, - }} - /> - - - - )} - - {!isDiagnosticsView && ( - <> - - - What needs attention - - - -
- - Supported target rate - - } - value={fmt(h.supported_target_rate, { pct: true })} - detail="share of oracle targets Microplex can evaluate" - /> - - Target win rate - - } - value={fmt(h.target_win_rate, { pct: true })} - detail="share beating us-data baseline in public summary" - /> - - Synthetic records - - } - value={fmt(h.n_synthetic)} - detail={`profile ${h.calibration_target_profile ?? "unknown"}`} - /> -
- -
-
-
- - Recurring regressing families - - - Families that often appear among the top regressions. - -
- []} - /> -
-
-
- - Repeated target flags - - - Targets recurring in audited regressions. - -
- []} - /> -
-
- - - - - - Use this to see whether recent Microplex runs are improving and which - target families dominate failures. Loss delta is candidate minus - us-data baseline; positive means Microplex is worse on that score. - - -
- - - Best public runs - - - - Lowest loss delta across{" "} - {fmt(data.regression_summary.total_scored_runs)} scored runs ( - {fmt(data.regression_summary.total_audited_runs)} audited). - - [] - } - /> - - - - - Worst public runs - - - [] - } - /> - - -
- - - The public drilldown is focused on{" "} - {data.irs_drilldown.family}, - the family leading {data.irs_drilldown.audits_where_family_leads}{" "} - audited regressions. Positive weighted delta means Microplex is worse - than the us-data baseline on that target. - - - - - Largest IRS target gaps - - - [] - } - /> - - - -
- - - Filing-status count gaps - - - [] - } - /> - - - - - MFS AGI-bin count gaps - - - [] - } - /> - - -
- - - - us-data baseline and artifact context - - - -
- - Composite parity loss - - } - value={fmt(h.candidate_composite_parity_loss)} - detail={`us-data baseline ${fmt( - h.baseline_composite_parity_loss, - )}`} - badge={deltaBadge(h.composite_parity_loss_delta)} - /> - - Mean |relative error| - - } - value={fmt(h.candidate_mean_abs_relative_error)} - detail={`us-data baseline ${fmt( - h.baseline_mean_abs_relative_error, - )}`} - badge={deltaBadge(h.mean_abs_relative_error_delta)} - /> - - Slice win rate - - } - value={fmt(h.slice_win_rate, { pct: true })} - detail={`artifact ${data.artifact_id ?? "unknown"}`} - /> -
- - {verdictEntries.length > 0 && ( -
- {verdictEntries.map(([key, passed]) => ( - - {passed ? "pass" : "fail"} {key} - - ))} -
- )} - -
- {data.source_artifacts.map((artifact) => ( - - {artifact.name} - - ))} -
-
    - {data.limitations.map((item) => ( -
  • {item}
    • - ))} -
- - - - - )} - - - ); -} diff --git a/frontend/app/microplex/reforms/page.tsx b/frontend/app/microplex/reforms/page.tsx deleted file mode 100644 index 7b4f28e..0000000 --- a/frontend/app/microplex/reforms/page.tsx +++ /dev/null @@ -1 +0,0 @@ -export { default } from "@/components/microplex/reform-benchmarks-page"; diff --git a/frontend/app/nodes/page.tsx b/frontend/app/nodes/page.tsx deleted file mode 100644 index c9542a5..0000000 --- a/frontend/app/nodes/page.tsx +++ /dev/null @@ -1,186 +0,0 @@ -"use client"; - -import { - Badge, - Card, - CardContent, - CardHeader, - CardTitle, - Input, - Stack, - Text, - Title, - formatNumber, -} from "@policyengine/ui-kit"; -import { useMemo, useState } from "react"; - -import { AppShell } from "@/components/layout/app-shell"; -import { DataTable } from "@/components/shared/InteractiveDataTable"; -import { LoadingBlock } from "@/components/shared/LoadingBlock"; -import { useNodes, type NodeVariable } from "@/lib/api/hooks/use-nodes"; - -type CalFilter = "all" | "calibrated" | "uncalibrated"; -type EntityFilter = "all" | string; - -const columns = [ - { key: "name", header: "Variable", sortable: true }, - { - key: "entity", - header: "Entity", - sortable: true, - format: (val: unknown) => ( - {String(val)} - ), - }, - { - key: "value_type", - header: "Type", - sortable: true, - format: (val: unknown) => ( - {String(val)} - ), - }, - { - key: "is_calibrated", - header: "Calibrated?", - sortable: true, - format: (val: unknown) => - val ? ( - calibrated - ) : ( - uncalibrated - ), - }, - { - key: "label", - header: "Label", - format: (val: unknown) => ( - {String(val)} - ), - }, -]; - -function NodesTable() { - const { data, isLoading, error } = useNodes(); - const [search, setSearch] = useState(""); - const [calFilter, setCalFilter] = useState("all"); - const [entityFilter, setEntityFilter] = useState("all"); - - const entityOptions = useMemo(() => { - if (!data) return []; - return Array.from(new Set(data.items.map((r) => r.entity))).sort(); - }, [data]); - - const filtered = useMemo(() => { - if (!data) return []; - const q = search.trim().toLowerCase(); - return data.items.filter((row: NodeVariable) => { - if (calFilter === "calibrated" && !row.is_calibrated) return false; - if (calFilter === "uncalibrated" && row.is_calibrated) return false; - if (entityFilter !== "all" && row.entity !== entityFilter) return false; - if (!q) return true; - return ( - row.name.toLowerCase().includes(q) || - row.label.toLowerCase().includes(q) - ); - }); - }, [data, search, calFilter, entityFilter]); - - if (isLoading) return ; - if (error) - return ( - - Failed to load node variables: {String(error)} - - ); - if (!data) return ; - - return ( - -
-
- setSearch(e.target.value)} - /> -
-
- - Status - - -
-
- - Entity - - -
-
- - - Showing {formatNumber(filtered.length)} of{" "} - {formatNumber(data.total)} leaf variables ·{" "} - {formatNumber(data.n_calibrated)} calibrated overall ( - {((data.n_calibrated / Math.max(data.total, 1)) * 100).toFixed(1)}%) - - - - - ); -} - -export default function NodesPage() { - return ( - - -
- Node variables - - Leaf inputs in the policyengine_us variable tree — - variables with no formula and no adds/ - subtracts. (Uprating is allowed; it's just - CPI/wage projection of a stored value, not derivation.) These - can't be computed by the microsim and must come from the - underlying dataset or from a calibration target. Use this view to - see which leaves carry a target in the loaded run and which - don't. - -
- - - - Leaves - - - - - - - - ); -} diff --git a/frontend/app/page.tsx b/frontend/app/page.tsx index a121153..c558587 100644 --- a/frontend/app/page.tsx +++ b/frontend/app/page.tsx @@ -1,5 +1,5 @@ import { redirect } from "next/navigation"; export default function Page() { - redirect("/microplex"); + redirect("/populace"); } diff --git a/frontend/app/pipeline/page.tsx b/frontend/app/pipeline/page.tsx deleted file mode 100644 index 29a196c..0000000 --- a/frontend/app/pipeline/page.tsx +++ /dev/null @@ -1,381 +0,0 @@ -"use client"; - -import { useEffect, useMemo, useState } from "react"; -import ReactMarkdown from "react-markdown"; -import { - Card, - CardContent, - CardHeader, - CardTitle, - Skeleton, - Stack, - Title, - Text, - Badge, - formatNumber, -} from "@policyengine/ui-kit"; -import { AppShell } from "@/components/layout/app-shell"; -import { - PIPELINE_OPTIONS, - usePipeline, - useStageDoc, - type PipelineNode, - type PipelineStage, -} from "@/lib/api/hooks/use-pipeline"; -import { useDashboardMode } from "@/lib/dashboard-mode-context"; -import { PipelineGraph } from "@/components/pipeline/pipeline-graph"; - -const STATUS_VARIANT: Record = { - current: "success", - transitional: "warning", - legacy: "secondary", - planned: "secondary", - unknown: "secondary", -}; - -function StageCard({ - stage, - active, - onClick, -}: { - stage: PipelineStage; - active: boolean; - onClick: () => void; -}) { - return ( - - ); -} - -function NodeDetail({ node }: { node: PipelineNode }) { - return ( -
-
- {node.id} - - {node.status ?? "?"} - - {node.node_type} - {(node.pathways ?? []).map((p) => ( - - {p} - - ))} -
- {node.label &&
{node.label}
} - {node.description && ( -

{node.description}

- )} - {node.explanation && ( -
- {node.explanation} -
- )} -
- - {node.source_file?.replace("policyengine_us_data/", "")} - {node.decorator_line ? `:${node.decorator_line}` : ""} - -
- {node.artifacts_in && node.artifacts_in.length > 0 && ( -
- in:{" "} - - {node.artifacts_in.join(", ")} - -
- )} - {node.artifacts_out && node.artifacts_out.length > 0 && ( -
- out:{" "} - - {node.artifacts_out.join(", ")} - -
- )} - {node.implementation_refs && node.implementation_refs.length > 0 && ( -
- implementation:{" "} - - {node.implementation_refs.join(", ")} - -
- )} - {node.validation_commands && node.validation_commands.length > 0 && ( -
- validation:{" "} - - {node.validation_commands.join(", ")} - -
- )} - {node.analyst_questions && node.analyst_questions.length > 0 && ( -
- questions to ask: -
    - {node.analyst_questions.map((question) => ( -
  • {question}
    • - ))} -
-
- )} -
- ); -} - -function StageDoc({ - stageId, - pipelineId, -}: { - stageId: string; - pipelineId: string; -}) { - const q = useStageDoc(stageId, pipelineId); - if (q.isLoading) return ; - if (q.error) - return ( - - Deep-dive for {stageId} not available yet. - - ); - if (!q.data) return null; - return ( -
- {q.data.markdown} -
- ); -} - -export default function PipelinePage() { - const { mode } = useDashboardMode(); - const [selectedPipelineId, setSelectedPipelineId] = useState( - mode === "us-data" ? "us-data" : "microplex-us", - ); - const pipeline = usePipeline(selectedPipelineId); - const [activeStage, setActiveStage] = useState(null); - const [selectedNodeId, setSelectedNodeId] = useState(null); - const [showIsolated, setShowIsolated] = useState(false); - - useEffect(() => { - setActiveStage(null); - setSelectedNodeId(null); - setShowIsolated(false); - }, [selectedPipelineId]); - - useEffect(() => { - setSelectedPipelineId(mode === "us-data" ? "us-data" : "microplex-us"); - }, [mode]); - - const selectedNode = useMemo(() => { - if (!pipeline.data || !selectedNodeId) return null; - return pipeline.data.nodes.find((n) => n.id === selectedNodeId) ?? null; - }, [pipeline.data, selectedNodeId]); - - return ( - - -
- Data pipeline - - Compare the extracted policyengine_us_data DAG with a - curated Microplex-US flow. Select a pipeline, click a stage to - filter, then click a node for its explanation. - -
- - - -
- - - - { - PIPELINE_OPTIONS.find((option) => option.id === selectedPipelineId) - ?.description - } - -
- - - - {pipeline.isLoading && } - {pipeline.error && ( - - - - Failed to load pipeline: {String(pipeline.error)} - - - Run{" "} - python backend/scripts/extract_pipeline_dag.py{" "} - if this is a fresh setup. - - - - )} - - {pipeline.data && ( - <> - - -
-
- - {pipeline.data.pipeline_label ?? selectedPipelineId} - - - {formatNumber(pipeline.data.stats.node_count)} nodes - - - {formatNumber(pipeline.data.stats.edge_count)} edges - - {pipeline.data.source_repo && ( - {pipeline.data.source_repo} - )} -
- {pipeline.data.description && ( - - {pipeline.data.description} - - )} - {pipeline.data.source_urls && pipeline.data.source_urls.length > 0 && ( -
- {pipeline.data.source_urls.map((url) => ( - - {new URL(url).pathname.split("/").slice(-1)[0] || url} - - ))} -
- )} -
- - - -
- {pipeline.data.stages.map((s) => ( - { - setActiveStage(activeStage === s.id ? null : s.id); - setSelectedNodeId(null); - }} - /> - ))} -
- - - -
- - Graph - {activeStage && ( - - filtered to {activeStage} - - )} - - -
- - - - - Showing only nodes that participate in declared data flow ( - {pipeline.data.edges.length} edges). Many pipeline nodes - don't declare formal artifacts_in/out{" "} - metadata and are hidden by default — toggle{" "} - Show isolated nodes to see all{" "} - {pipeline.data.stats.node_count}. - - - - - {selectedNode && ( - - - - Node detail - - - - - - - - )} - - {activeStage && ( - - - Deep dive: {activeStage} - - - - - - )} - - )} - - - ); -} diff --git a/frontend/app/populace/compare/page.tsx b/frontend/app/populace/compare/page.tsx new file mode 100644 index 0000000..b8cfd7b --- /dev/null +++ b/frontend/app/populace/compare/page.tsx @@ -0,0 +1,10 @@ +import { AppShell } from "@/components/layout/app-shell"; +import { PopulaceCompareView } from "@/components/populace/populace-compare-view"; + +export default function PopulaceComparePage() { + return ( + + + + ); +} diff --git a/frontend/app/populace/page.tsx b/frontend/app/populace/page.tsx new file mode 100644 index 0000000..43dc324 --- /dev/null +++ b/frontend/app/populace/page.tsx @@ -0,0 +1,10 @@ +import { AppShell } from "@/components/layout/app-shell"; +import { PopulaceOverviewView } from "@/components/populace/populace-overview-view"; + +export default function PopulaceOverviewPage() { + return ( + + + + ); +} diff --git a/frontend/app/populace/targets/page.tsx b/frontend/app/populace/targets/page.tsx new file mode 100644 index 0000000..ee25188 --- /dev/null +++ b/frontend/app/populace/targets/page.tsx @@ -0,0 +1,10 @@ +import { AppShell } from "@/components/layout/app-shell"; +import { PopulaceTargetsView } from "@/components/populace/populace-targets-view"; + +export default function PopulaceTargetsPage() { + return ( + + + + ); +} diff --git a/frontend/app/providers.tsx b/frontend/app/providers.tsx index 2b427bd..4b57a2b 100644 --- a/frontend/app/providers.tsx +++ b/frontend/app/providers.tsx @@ -2,10 +2,7 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; -import { Suspense, useState } from "react"; -import { DashboardModeProvider } from "@/lib/dashboard-mode-context"; -import { GeoProvider } from "@/lib/geo-context"; -import { RunProvider } from "@/lib/run-context"; +import { useState } from "react"; export function Providers({ children }: { children: React.ReactNode }) { const [queryClient] = useState( @@ -23,13 +20,7 @@ export function Providers({ children }: { children: React.ReactNode }) { return ( - - - - {children} - - - + {children} ); diff --git a/frontend/app/summary/page.tsx b/frontend/app/summary/page.tsx deleted file mode 100644 index fc978f2..0000000 --- a/frontend/app/summary/page.tsx +++ /dev/null @@ -1,216 +0,0 @@ -"use client"; - -import Link from "next/link"; -import { - Card, - CardHeader, - CardTitle, - CardContent, - Skeleton, - Stack, - Title, - Text, - Badge, - formatNumber, - PEBarChart, - ChartContainer, -} from "@policyengine/ui-kit"; -import { AppShell } from "@/components/layout/app-shell"; -import { useSummary, type SummaryResponse } from "@/lib/api/hooks/use-summary"; -import { useRunContext } from "@/lib/run-context"; - -function pct(v: number | null | undefined, digits = 1): string { - if (v === null || v === undefined || !isFinite(v)) return "—"; - return `${(v * 100).toFixed(digits)}%`; -} - -function num(v: number | null | undefined): string { - if (v === null || v === undefined || !isFinite(v)) return "—"; - return formatNumber(v); -} - -function Kpi({ - label, - value, - hint, -}: { - label: string; - value: string; - hint?: string; -}) { - return ( -
- - {label} - - {value} - {hint && ( - - {hint} - - )} -
- ); -} - -function Scorecard({ data }: { data: SummaryResponse }) { - const h = data.headline; - return ( -
- - - - -
- ); -} - -function ErrorDistribution({ data }: { data: SummaryResponse }) { - const fmt = (v: number) => `${Math.round(v * 100)}%`; - const bins = data.error_distribution.map((b) => ({ - range: b.overflow ? `>${fmt(b.bin_min)}` : `${fmt(b.bin_min)}–${fmt(b.bin_max)}`, - count: b.count, - })); - if (bins.length === 0) { - return No targets to plot.; - } - return ( - - ); -} - -function WorstTargets({ rows }: { rows: SummaryResponse["worst_targets"] }) { - if (rows.length === 0) { - return No targets to show.; - } - return ( - - - - - - - - - - - {rows.map((r) => ( - - - - - - - ))} - -
TargetTarget valueEstimateRel. error
- - {r.target_name} - - {num(r.value)}{num(r.estimate)} - {r.rel_error == null || r.abs_rel_error == null ? ( - — - ) : ( - 0.25 ? "destructive" : "secondary"}> - {r.rel_error >= 0 ? "+" : ""} - {(r.rel_error * 100).toFixed(1)}% - - )} -
- ); -} - -export default function Page() { - const { dataset, run } = useRunContext(); - const summary = useSummary(); - - return ( - - -
- Run summary - - {dataset && run ? ( - <> - Dataset {dataset} · Run{" "} - {run} - - ) : ( - "Select a dataset and run above to load diagnostics." - )} - -
- - {summary.isLoading && ( - <> - - - - - )} - - {summary.error && ( - - - - Failed to load summary: {String(summary.error)} - - - - )} - - {summary.data && ( - <> - - - - - Absolute relative error distribution - - - - - - - - - - - - Top 10 worst-fit targets - - - - - - - - )} - - - ); -} diff --git a/frontend/app/targets/page.tsx b/frontend/app/targets/page.tsx deleted file mode 100644 index 375a263..0000000 --- a/frontend/app/targets/page.tsx +++ /dev/null @@ -1,622 +0,0 @@ -"use client"; - -import { - Card, - CardHeader, - CardTitle, - CardContent, - Badge, - Tabs, - TabsList, - TabsTrigger, - TabsContent, - Skeleton, - Alert, - AlertTitle, - AlertDescription, - Stack, - Group, - Title, - Text, - formatPercent, - formatNumber, - MetricCard, -} from "@policyengine/ui-kit"; -import { DataTable } from "@/components/shared/InteractiveDataTable"; -import { LoadingBlock } from "@/components/shared/LoadingBlock"; -import { AppShell } from "@/components/layout/app-shell"; -import { useTargets } from "@/lib/api/hooks/use-targets"; -import { useSummary } from "@/lib/api/hooks/use-summary"; -import { - useErrorDecomposition, - useConstraintDiff, - useContributors, - useTargetConvergence, - useProvenance, -} from "@/lib/api/hooks/use-target-detail"; -import { - TargetFiltersProvider, - statusToIncludedOnly, - useTargetFilters, -} from "@/lib/target-filters-context"; -import { TargetChipBar } from "@/components/targets/chip-bar"; -import { TargetSearchAndControls } from "@/components/targets/search-and-controls"; -import { TargetPagination } from "@/components/targets/pagination"; -import { RunSelectorCard } from "@/components/targets/run-selector-card"; -import { CompareProvider, useCompareMode } from "@/lib/compare-context"; -import { STATE_FIPS_TO_CODE } from "@/lib/geo-names"; - -/** - * Map a target's (geo_level, geographic_id) to the output dataset file the - * pipeline builds for it. e.g. district 0612 → "districts/CA-12.h5". - */ -function datasetForRow(row: { - geo_level?: string | null; - geographic_id?: string | null; -}): string { - const level = row.geo_level ?? ""; - if (level === "national") return "national/US.h5"; - const gid = row.geographic_id ?? ""; - if (!gid) return "—"; - if (level === "state") { - const n = parseInt(gid, 10); - const code = Number.isFinite(n) ? STATE_FIPS_TO_CODE[n] : null; - return `states/${code ?? gid}.h5`; - } - if (level === "district") { - const n = parseInt(gid, 10); - if (Number.isFinite(n)) { - const state = STATE_FIPS_TO_CODE[Math.floor(n / 100)] ?? String(Math.floor(n / 100)); - const dist = String(n % 100).padStart(2, "0"); - return `districts/${state}-${dist}.h5`; - } - return `districts/${gid}.h5`; - } - return level; -} -import Link from "next/link"; -import { useSearchParams, useRouter } from "next/navigation"; -import { Suspense } from "react"; - -/** Render free-text notes, linkifying URLs. */ -function NotesWithLinks({ notes }: { notes: string }) { - const parts = notes.split(/(https?:\/\/\S+)/g); - return ( - <> - {parts.map((p, i) => - /^https?:\/\//.test(p) ? ( - - {p} - - ) : ( - {p} - ), - )} - - ); -} - -function buildTargetColumns(compareOn: boolean) { - const base = baseTargetColumns; - if (!compareOn) return base; - // Splice compare-only columns immediately after the rel_error column so - // the user reads A → B → Δ left-to-right next to the existing PE - // aggregate / Rel. error pair. - const relIdx = base.findIndex((c) => c.key === "rel_error"); - return [ - ...base.slice(0, relIdx + 1), - { - key: "estimate_b", - header: "PE agg (B)", - align: "right" as const, - format: (val: unknown) => - val == null - ? — - : formatNumber(Number(val)), - }, - { - key: "rel_error_b", - header: "Rel. error (B)", - align: "right" as const, - format: (val: unknown) => { - if (val == null) return —; - const v = Number(val); - const abs = Math.abs(v); - const variant = - abs > 0.5 ? "error" : abs > 0.2 ? "warning" : abs > 0.05 ? "secondary" : "success"; - const display = abs >= 1 ? `${(v * 100).toFixed(0)}%` : `${(v * 100).toFixed(1)}%`; - return {display}; - }, - }, - { - key: "delta", - header: "Δ |err|", - align: "right" as const, - format: (val: unknown) => { - if (val == null || !Number.isFinite(Number(val))) { - return —; - } - const v = Number(val); - // Negative delta = B improved (lower |err|); positive = regressed. - const variant: "success" | "error" | "secondary" = - Math.abs(v) < 1e-6 ? "secondary" : v < 0 ? "success" : "error"; - const sign = v > 0 ? "+" : ""; - return {sign}{(v * 100).toFixed(1)}pp; - }, - }, - ...base.slice(relIdx + 1), - ]; -} - -const baseTargetColumns = [ - { - key: "target_id", - header: "ID", - format: (val: unknown) => - val != null ? `#${val}` : —, - }, - { - key: "geo_display_name", - header: "Geography", - format: (val: unknown) => String(val ?? "National"), - }, - { - key: "dataset", - header: "Dataset", - format: (_val: unknown, row: Record) => ( - - {datasetForRow(row as never)} - - ), - }, - { - key: "variable", - header: "Variable", - format: (val: unknown, row: Record) => { - const constraints = (row.constraints as string[] | undefined) ?? []; - const sub = - constraints.length === 0 - ? "all population" - : constraints.join(", "); - return ( -
- {String(val)} - · {sub} -
- ); - }, - }, - { - key: "target_value", - header: "Target", - align: "right" as const, - format: (val: unknown) => formatNumber(Number(val)), - }, - { - key: "estimate", - header: "PE aggregate", - align: "right" as const, - format: (val: unknown) => - val == null - ? — - : formatNumber(Number(val)), - }, - { - key: "rel_error", - header: "Rel. error", - align: "right" as const, - format: (val: unknown) => { - if (val == null) return —; - const v = Number(val); - const abs = Math.abs(v); - const variant = - abs > 0.5 - ? "error" - : abs > 0.2 - ? "warning" - : abs > 0.05 - ? "secondary" - : "success"; - const display = abs >= 1 ? `${(v * 100).toFixed(0)}%` : `${(v * 100).toFixed(1)}%`; - return {display}; - }, - }, - { - key: "source", - header: "Source", - format: (val: unknown) => - val == null || val === "" ? ( - — - ) : ( - {String(val)} - ), - }, - { - key: "included", - header: "Status", - format: (val: unknown) => - val ? ( - In loss - ) : ( - Not in loss - ), - }, -]; - -const contributorColumns = [ - { - key: "household_idx", - header: "Household", - format: (val: unknown) => ( - - #{String(val)} - - ), - }, - { - key: "g_weight", - header: "G-weight", - align: "right" as const, - format: (val: unknown) => Number(val).toFixed(1), - }, - { - key: "raw_value", - header: "Value", - align: "right" as const, - format: (val: unknown) => Number(val).toLocaleString(), - }, - { - key: "income", - header: "Income", - align: "right" as const, - format: (val: unknown) => `$${Number(val).toLocaleString()}`, - }, - { - key: "in_poverty", - header: "Poverty", - align: "center" as const, - format: (val: unknown) => - val ? ( - Yes - ) : ( - - No - - ), - }, -]; - -const constraintColumns = [ - { - key: "constraint", - header: "Constraint", - format: (_: unknown, row: Record) => - `${row.variable} ${row.operation} ${row.value}`, - }, - { - key: "contributors_satisfying", - header: "Satisfying", - align: "right" as const, - format: (val: unknown) => Number(val).toLocaleString(), - }, - { - key: "contributors_violating", - header: "Violating", - align: "right" as const, - format: (val: unknown) => Number(val).toLocaleString(), - }, - { - key: "status", - header: "Status", - align: "right" as const, - format: (val: unknown) => { - const variant = - val === "OK" - ? "success" - : val === "MINOR_VIOLATION" - ? "warning" - : "error"; - return {String(val)}; - }, - }, -]; - -function TargetTable() { - const { filters, setFilters } = useTargetFilters(); - const { enabled: compareOn, runB } = useCompareMode(); - const searchParams = useSearchParams(); - const selectedIdx = searchParams.get("selected") - ? Number(searchParams.get("selected")) - : null; - - const compareRun = compareOn ? runB : null; - const targets = useTargets({ - sortBy: filters.sortBy, - sortOrder: filters.sortOrder, - search: filters.search, - variables: filters.variables, - geoLevels: filters.geoLevels, - errorBuckets: filters.errorBuckets, - stateFips: - filters.stateFipsList.length > 0 ? filters.stateFipsList : undefined, - sources: filters.sources.length > 0 ? filters.sources : undefined, - datasetFiles: - filters.datasetFiles.length > 0 ? filters.datasetFiles : undefined, - includedOnly: statusToIncludedOnly(filters.status), - compareRun, - limit: filters.pageSize, - offset: filters.page * filters.pageSize, - }); - - const columns = buildTargetColumns(!!compareRun); - - const bundleEvaluated = targets.data?.bundle_evaluated; - - return ( - - {bundleEvaluated && ( -
- Bundle-fit numbers. PE aggregates below were - evaluated against{" "} - {bundleEvaluated}{" "} - — the calibrated h5 the pipeline builds for this bundle — not - the federal enhanced_cps_2024.h5. -
- )} - {targets.data ? ( - { - if (s) { - setFilters({ - sortBy: s.key as typeof filters.sortBy, - sortOrder: s.direction, - }); - } - }} - data={targets.data.items.map((t) => ({ - ...t, - _selected: t.target_idx === selectedIdx, - }))} - /> - ) : targets.error ? ( -
- Failed to load targets: {String(targets.error)} -
- ) : ( - - )} - - - ); -} - -function DetailPanel() { - const searchParams = useSearchParams(); - const router = useRouter(); - const selectedIdx = searchParams.get("selected") - ? Number(searchParams.get("selected")) - : null; - - const errorDecomp = useErrorDecomposition(selectedIdx); - const constraintDiff = useConstraintDiff(selectedIdx); - const contributors = useContributors(selectedIdx, { limit: 10 }); - const convergence = useTargetConvergence(selectedIdx); - const provenance = useProvenance(selectedIdx); - - if (selectedIdx === null) return null; - - return ( - - - - Target detail: #{selectedIdx} - - - - - - - Overview - Convergence - Contributors - Constraints - - - - - {errorDecomp.data && ( - - Error decomposition - - - - - - - - Diagnosis - - {errorDecomp.data.diagnosis} - - - - )} - - {provenance.data && ( - - Provenance - - - Source: {provenance.data.source} - - - Period: {provenance.data.period} - - {provenance.data.tolerance && ( - - Tolerance: {provenance.data.tolerance}% - - )} - - {provenance.data.notes && ( -
- Notes: - -
- )} - - - Constraints: - - {provenance.data.constraints.map((c, i) => ( - - {c.variable} {c.operation} {c.value} - - ))} - - - )} - - - - - {convergence.data && convergence.data.length > 0 ? ( - - - {convergence.data.length} epoch checkpoints. Final rel_error:{" "} - {formatPercent( - convergence.data[convergence.data.length - 1].rel_error, - 1, - )} - - Number(v).toExponential(2), - }, - { - key: "rel_error", - header: "Rel. error", - align: "right" as const, - format: (v: unknown) => formatPercent(Number(v), 1), - }, - ]} - data={convergence.data} - /> - - ) : ( - - No convergence data available - - )} - - - - {contributors.data ? ( - - ) : ( - - )} - - - - {constraintDiff.data ? ( - - ) : ( - - )} - - - - - ); -} - -function TargetExplorerContent() { - return ( - - -
- All targets - - Every target known to policy_data.db. Status shows - whether the active calibration uses it; Dataset shows which output - bundle the pipeline builds it into. - -
- - - -
- - - - -
- - - ); -} - -export default function TargetExplorerPage() { - return ( - - - - - - - - ); -} diff --git a/frontend/app/weights/page.tsx b/frontend/app/weights/page.tsx deleted file mode 100644 index 4dbbe2c..0000000 --- a/frontend/app/weights/page.tsx +++ /dev/null @@ -1,315 +0,0 @@ -"use client"; - -import { - Card, - CardHeader, - CardTitle, - CardContent, - MetricCard, - SegmentedControl, - PEBarChart, - ChartContainer, - Skeleton, - Stack, - Group, - Title, - Tooltip, - TooltipTrigger, - TooltipContent, - TooltipProvider, - formatNumber, -} from "@policyengine/ui-kit"; -import { DataTable } from "@/components/shared/InteractiveDataTable"; -import { AppShell } from "@/components/layout/app-shell"; -import { - useWeightDistribution, - useWeightHistogram, -} from "@/lib/api/hooks/use-weights"; -import { useGeo, useGeoParams } from "@/lib/geo-context"; -import { useState } from "react"; - -function TipHeader({ label, tip }: { label: string; tip: string }) { - return ( - - - - {label} - - -

{tip}

- - - - ); -} - -function colorByDeviation(val: number, baseline: number): string { - const ratio = Math.abs(val - baseline) / Math.max(baseline, 0.001); - if (ratio > 2) return "bg-red-100 text-red-900"; - if (ratio > 1) return "bg-orange-100 text-orange-900"; - if (ratio > 0.5) return "bg-yellow-50 text-yellow-900"; - return ""; -} - -export default function WeightsPage() { - const [sliceBy, setSliceBy] = useState("income_decile"); - const [metric, setMetric] = useState("g_weight"); - - const { geo } = useGeo(); - const geoParams = useGeoParams(); - const distribution = useWeightDistribution({ sliceBy, metric, ...geoParams }); - const decileWeights = useWeightDistribution({ - sliceBy: "income_decile", - metric: "final_weight", - ...geoParams, - }); - const histogram = useWeightHistogram({ metric, logScale: true, ...geoParams }); - - const histogramChartData = - histogram.data?.map((bin) => ({ - range: `${bin.bin_min.toFixed(2)}-${bin.bin_max.toFixed(0)}`, - count: bin.count, - })) ?? []; - - return ( - - - Weight landscape - - {/* Controls */} - - - Metric - - - - Slice by - - - - - {/* Stats cards */} - {distribution.data ? ( - - - - -
- -
- - -

Kish effective N: how many equally-weighted records this dataset is equivalent to. Lower means more information lost to unequal weighting.

- - - - -
- -
- - -

Coefficient of variation of weights (std / mean). CV = 0 means all weights equal. Higher means more extreme weight variation. Values above 1 indicate significant distortion.

- - - - -
- -
- - -

Equal to 1 + CV². Measures how much unequal weighting inflates variance. A design effect of 5 means you'd need 5x fewer equally-weighted records for the same precision.

- - - - -
- -
- - -

Share of total population weight held by the heaviest 1% of records. High values mean a few records dominate all weighted statistics.

- - - - -
- -
- - -

Share of total population weight held by the heaviest 5% of records. In a healthy calibration this should be under 20%.

- - - - - ) : ( - - {Array.from({ length: 5 }).map((_, i) => ( - - ))} - - )} - - {/* Histogram */} - {histogram.data ? ( - - - - ) : ( - - )} - - {/* Weighted population by income decile */} - {decileWeights.data && decileWeights.data.slices.length > 0 && ( - - ({ - decile: s.label, - weighted_population: Math.round(s.n * s.mean), - }))} - xKey="decile" - yKey="weighted_population" - height={300} - fillColor="var(--chart-2)" - /> - - )} - - {/* Slices */} - {distribution.data && distribution.data.slices.length > 0 && (() => { - const slices = distribution.data!.slices; - const overallMean = distribution.data!.mean; - const overallMedian = distribution.data!.median; - - const metricLabel = metric === "g_weight" ? "g-weight" : metric.replace("_", " "); - const sliceColumns = [ - { - key: "label", - header: "Group", - }, - { - key: "n", - header: , - align: "right" as const, - format: (v: unknown) => Number(v).toLocaleString(), - }, - { - key: "weighted_pop", - header: , - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "kish_effective_n", - header: , - align: "right" as const, - format: (v: unknown) => formatNumber(Number(v)), - }, - { - key: "efficiency", - header: , - align: "right" as const, - format: (v: unknown) => { - const eff = Number(v); - const cls = eff < 0.1 ? "bg-red-100 text-red-900 px-2 py-0.5 rounded" : - eff < 0.25 ? "bg-orange-100 text-orange-900 px-2 py-0.5 rounded" : - eff < 0.5 ? "bg-yellow-50 text-yellow-900 px-2 py-0.5 rounded" : ""; - return {(eff * 100).toFixed(1)}%; - }, - }, - { - key: "mean", - header: 2x off), orange (>1x), yellow (>0.5x).`} />, - align: "right" as const, - format: (v: unknown) => { - const val = Number(v); - const cls = colorByDeviation(val, overallMean); - return {val.toFixed(3)}; - }, - }, - { - key: "median", - header: , - align: "right" as const, - format: (v: unknown) => { - const val = Number(v); - const cls = colorByDeviation(val, overallMedian); - return {val.toFixed(3)}; - }, - }, - ]; - - const enrichedSlices = slices.map((s) => ({ - ...s, - weighted_pop: Math.round(s.n * s.mean), - efficiency: s.n > 0 ? s.kish_effective_n / s.n : 0, - })); - - return ( - - - Weight analysis by group - - -
- -
- - - ); - })()} - - - ); -} diff --git a/frontend/bun.lock b/frontend/bun.lock index b312675..16584b2 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -19,6 +19,7 @@ }, "devDependencies": { "@tailwindcss/postcss": "^4.2.0", + "@types/bun": "^1.3.14", "@types/node": "^22.0.0", "@types/react": "^19.2.0", "@types/react-dom": "^19.2.0", @@ -295,6 +296,8 @@ "@tanstack/react-query-devtools": ["@tanstack/react-query-devtools@5.95.2", "", { "dependencies": { "@tanstack/query-devtools": "5.95.2" }, "peerDependencies": { "@tanstack/react-query": "^5.95.2", "react": "^18 || ^19" } }, "sha512-AFQFmbznVkbtfpx8VJ2DylW17wWagQel/qLstVLkYmNRo2CmJt3SNej5hvl6EnEeljJIdC3BTB+W7HZtpsH+3g=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/d3": ["@types/d3@7.4.3", "", { "dependencies": { "@types/d3-array": "*", "@types/d3-axis": "*", "@types/d3-brush": "*", "@types/d3-chord": "*", "@types/d3-color": "*", "@types/d3-contour": "*", "@types/d3-delaunay": "*", "@types/d3-dispatch": "*", "@types/d3-drag": "*", "@types/d3-dsv": "*", "@types/d3-ease": "*", "@types/d3-fetch": "*", "@types/d3-force": "*", "@types/d3-format": "*", "@types/d3-geo": "*", "@types/d3-hierarchy": "*", "@types/d3-interpolate": "*", "@types/d3-path": "*", "@types/d3-polygon": "*", "@types/d3-quadtree": "*", "@types/d3-random": "*", "@types/d3-scale": "*", "@types/d3-scale-chromatic": "*", "@types/d3-selection": "*", "@types/d3-shape": "*", "@types/d3-time": "*", "@types/d3-time-format": "*", "@types/d3-timer": "*", "@types/d3-transition": "*", "@types/d3-zoom": "*" } }, "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww=="], "@types/d3-array": ["@types/d3-array@3.2.2", "", {}, "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="], @@ -389,6 +392,8 @@ "baseline-browser-mapping": ["baseline-browser-mapping@2.10.29", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-Asa2krT+XTPZINCS+2QcyS8WTkObE77RwkydwF7h6DmnKqbvlalz93m/dnphUyCa6SWSP51VgtEUf2FN+gelFQ=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "caniuse-lite": ["caniuse-lite@1.0.30001782", "", {}, "sha512-dZcaJLJeDMh4rELYFw1tvSn1bhZWYFOt468FcbHHxx/Z/dFidd1I6ciyFdi3iwfQCyOjqo9upF6lGQYtMiJWxw=="], "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], diff --git a/frontend/components/layout/app-shell.tsx b/frontend/components/layout/app-shell.tsx index 37f43fb..0aab635 100644 --- a/frontend/components/layout/app-shell.tsx +++ b/frontend/components/layout/app-shell.tsx @@ -3,13 +3,11 @@ import { DashboardShell, Header } from "@policyengine/ui-kit"; import { GlobalLoader } from "./global-loader"; import { NavSidebar } from "./nav-sidebar"; -import { RunBootstrap } from "./run-bootstrap"; export function AppShell({ children }: { children: React.ReactNode }) { return ( -