From c70dbd7810133c92f85033775d8301c7557c961b Mon Sep 17 00:00:00 2001 From: Pavel Makarchuk Date: Mon, 15 Jun 2026 10:53:52 -0400 Subject: [PATCH] Publish incumbent-comparison scorecards (schema, pointer, validator, CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap that left the US incumbent comparison without a machine-readable artifact: the per-target populace-vs-enhanced-CPS result only survived on the older populace-us releases and was dropped from live populace in PolicyEngine/populace#37. - archive/us/populace-us-2024-9f1260b-20260611/scorecard.json: the first published scorecard (status "archived"), reconstructed from that release's sound_ecps_replacement_comparison.json — promotion metrics (full/holdout loss, unweighted MSRE), per-target win/loss/tie, per-family loss breakdown, top movers. - benchmarks/us/incumbent-comparison/latest.json: pointer to the most recent scorecard, so consumers resolve it without hard-coding a path. - scorecard.schema.json: the published contract (draft-07). - tools/validate_scorecard.py: stdlib validator — required keys, promotion metrics, and consistency the schema can't express (win counts sum to the target count; candidate_beats_baseline agrees with the losses). Verified it rejects both inconsistencies. - CI runs the validator on every PR. Fixes #3. Co-Authored-By: Claude Fable 5 --- .github/workflows/validate-scorecards.yml | 22 + README.md | 1 + .../scorecard.json | 1222 +++++++++++++++++ benchmarks/us/incumbent-comparison/README.md | 32 + .../us/incumbent-comparison/latest.json | 9 + .../scorecard.schema.json | 84 ++ tools/validate_scorecard.py | 170 +++ 7 files changed, 1540 insertions(+) create mode 100644 .github/workflows/validate-scorecards.yml create mode 100644 archive/us/populace-us-2024-9f1260b-20260611/scorecard.json create mode 100644 benchmarks/us/incumbent-comparison/latest.json create mode 100644 benchmarks/us/incumbent-comparison/scorecard.schema.json create mode 100644 tools/validate_scorecard.py diff --git a/.github/workflows/validate-scorecards.yml b/.github/workflows/validate-scorecards.yml new file mode 100644 index 0000000..36823c8 --- /dev/null +++ b/.github/workflows/validate-scorecards.yml @@ -0,0 +1,22 @@ +name: Validate scorecards + +# Published scorecards are a read-only contract for downstream consumers +# (e.g. the calibration-diagnostics dashboard). This gate runs the stdlib +# validator on every change so a malformed or internally-inconsistent +# scorecard or pointer cannot land on main. + +on: + push: + branches: [main] + pull_request: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Validate scorecards and pointers + run: python tools/validate_scorecard.py diff --git a/README.md b/README.md index 43b4e1b..a25351a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ historical candidate scorecards, and audit-only scripts belong here. | Area | Path | Purpose | | --- | --- | --- | | US incumbent comparison | `benchmarks/us/incumbent-comparison/` | Compare candidate Populace-US artifacts with the certified production incumbent on the frozen target surface. | +| Published scorecards | `archive/us//scorecard.json` + `benchmarks/us/incumbent-comparison/latest.json` | Machine-readable head-to-head results (schema + `latest.json` pointer) consumed by downstream dashboards. | | Historical archives | `archive/` | Small scorecards and run notes that document past candidate decisions. Large HDF5 artifacts stay in object storage or the local artifact cache. | ## Promotion Rule diff --git a/archive/us/populace-us-2024-9f1260b-20260611/scorecard.json b/archive/us/populace-us-2024-9f1260b-20260611/scorecard.json new file mode 100644 index 0000000..3a3a73b --- /dev/null +++ b/archive/us/populace-us-2024-9f1260b-20260611/scorecard.json @@ -0,0 +1,1222 @@ +{ + "schema_version": 1, + "candidate_release_id": "populace-us-2024-9f1260b-20260611", + "incumbent_manifest": "pinned-production-ecps-2024", + "period": 2024, + "metric": "sound_ecps_replacement_comparison", + "baseline_label": "enhanced_cps", + "candidate_label": "populace", + "protocol": "Matched 41,314 households, symmetric refit, 739-target holdout (seed 20260529).", + "provenance": { + "derived_from": "policyengine/populace-us releases/populace-us-2024-9f1260b-20260611/sound_ecps_replacement_comparison.json", + "note": "Archived incumbent comparison; recomputing requires the candidate and pinned-incumbent HDF5 artifacts." + }, + "summary": { + "candidate_loss": 0.227943, + "baseline_loss": 1.40549, + "loss_delta": -1.17755, + "candidate_holdout_loss": 0.0383735, + "baseline_holdout_loss": 0.316697, + "candidate_train_loss": 0.189569, + "baseline_train_loss": 1.08879, + "candidate_unweighted_msre": 0.23306, + "baseline_unweighted_msre": 1.32102, + "candidate_wins": 1040, + "baseline_wins": 2613, + "ties": 51, + "n_targets": 3704, + "holdout_targets": 739, + "train_targets": 2965, + "candidate_beats_baseline": true, + "matched_household_count": 41314 + }, + "family_breakdown": [ + { + "baseline_loss_contribution": 0.000202606, + "baseline_wins": 624, + "candidate_loss_contribution": 2.63978e-05, + "candidate_wins": 405, + "family": "national_irs_other", + "holdout_targets": 212, + "loss_delta": -0.000176208, + "n_targets": 1060, + "ties": 31, + "train_targets": 848 + }, + { + "baseline_loss_contribution": 0.000149663, + "baseline_wins": 758, + "candidate_loss_contribution": 1.89392e-05, + "candidate_wins": 260, + "family": "state_agi_distribution", + "holdout_targets": 204, + "loss_delta": -0.000130723, + "n_targets": 1019, + "ties": 1, + "train_targets": 815 + }, + { + "baseline_loss_contribution": 1.82942e-05, + "baseline_wins": 1, + "candidate_loss_contribution": 1.13257e-08, + "candidate_wins": 4, + "family": "national_tax_expenditures", + "holdout_targets": 1, + "loss_delta": -1.82829e-05, + "n_targets": 5, + "ties": 0, + "train_targets": 4 + }, + { + "baseline_loss_contribution": 4.89806e-06, + "baseline_wins": 719, + "candidate_loss_contribution": 9.79996e-06, + "candidate_wins": 176, + "family": "state_age_distribution", + "holdout_targets": 180, + "loss_delta": 4.9019e-06, + "n_targets": 900, + "ties": 5, + "train_targets": 720 + }, + { + "baseline_loss_contribution": 3.52091e-07, + "baseline_wins": 70, + "candidate_loss_contribution": 1.0432e-06, + "candidate_wins": 16, + "family": "national_population_by_age", + "holdout_targets": 17, + "loss_delta": 6.91108e-07, + "n_targets": 86, + "ties": 0, + "train_targets": 69 + }, + { + "baseline_loss_contribution": 1.71619e-06, + "baseline_wins": 232, + "candidate_loss_contribution": 2.26669e-06, + "candidate_wins": 111, + "family": "other", + "holdout_targets": 70, + "loss_delta": 5.50497e-07, + "n_targets": 348, + "ties": 5, + "train_targets": 278 + }, + { + "baseline_loss_contribution": 1.15573e-07, + "baseline_wins": 46, + "candidate_loss_contribution": 5.64995e-07, + "candidate_wins": 4, + "family": "state_population_under_5", + "holdout_targets": 10, + "loss_delta": 4.49421e-07, + "n_targets": 52, + "ties": 2, + "train_targets": 42 + }, + { + "baseline_loss_contribution": 3.37894e-08, + "baseline_wins": 39, + "candidate_loss_contribution": 3.5946e-07, + "candidate_wins": 12, + "family": "state_aca_enrollment", + "holdout_targets": 10, + "loss_delta": 3.25671e-07, + "n_targets": 51, + "ties": 0, + "train_targets": 41 + }, + { + "baseline_loss_contribution": 2.06017e-07, + "baseline_wins": 36, + "candidate_loss_contribution": 5.15114e-07, + "candidate_wins": 15, + "family": "state_snap_cost", + "holdout_targets": 10, + "loss_delta": 3.09096e-07, + "n_targets": 51, + "ties": 0, + "train_targets": 41 + }, + { + "baseline_loss_contribution": 7.38489e-07, + "baseline_wins": 9, + "candidate_loss_contribution": 1.01514e-06, + "candidate_wins": 8, + "family": "national_census_other", + "holdout_targets": 4, + "loss_delta": 2.76646e-07, + "n_targets": 22, + "ties": 5, + "train_targets": 18 + }, + { + "baseline_loss_contribution": 3.06769e-07, + "baseline_wins": 1, + "candidate_loss_contribution": 1.2182e-07, + "candidate_wins": 4, + "family": "national_ssa", + "holdout_targets": 1, + "loss_delta": -1.84949e-07, + "n_targets": 5, + "ties": 0, + "train_targets": 4 + }, + { + "baseline_loss_contribution": 4.62669e-07, + "baseline_wins": 31, + "candidate_loss_contribution": 3.73515e-07, + "candidate_wins": 20, + "family": "state_snap_households", + "holdout_targets": 10, + "loss_delta": -8.91541e-08, + "n_targets": 51, + "ties": 0, + "train_targets": 41 + }, + { + "baseline_loss_contribution": 5.94819e-08, + "baseline_wins": 45, + "candidate_loss_contribution": 1.25159e-07, + "candidate_wins": 5, + "family": "state_population", + "holdout_targets": 10, + "loss_delta": 6.56774e-08, + "n_targets": 52, + "ties": 2, + "train_targets": 42 + }, + { + "baseline_loss_contribution": 2.48211e-11, + "baseline_wins": 1, + "candidate_loss_contribution": 6.13654e-09, + "candidate_wins": 0, + "family": "national_infants", + "holdout_targets": 0, + "loss_delta": 6.11172e-09, + "n_targets": 1, + "ties": 0, + "train_targets": 1 + }, + { + "baseline_loss_contribution": 4.86085e-11, + "baseline_wins": 1, + "candidate_loss_contribution": 7.48223e-11, + "candidate_wins": 0, + "family": "national_net_worth", + "holdout_targets": 0, + "loss_delta": 2.62138e-11, + "n_targets": 1, + "ties": 0, + "train_targets": 1 + } + ], + "top_improvements": [ + { + "baseline_abs_scaled_error": 0.479335, + "baseline_error": 0.479335, + "baseline_estimate": 0.499563, + "baseline_loss_term": 0.229762, + "baseline_relative_error": 1.45, + "candidate_abs_scaled_error": 0.00202215, + "candidate_error": 0.00202215, + "candidate_estimate": 0.02225, + "candidate_loss_term": 4.08908e-06, + "candidate_relative_error": 0.982155, + "family": "national_irs_other", + "loss_delta": -0.229758, + "split": "train", + "target_index": 576, + "target_name": "nation/irs/adjusted gross income/total/AGI in 79m-inf/taxable/All", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.374737, + "baseline_error": 0.374737, + "baseline_estimate": 0.38893, + "baseline_loss_term": 0.140428, + "baseline_relative_error": 1.3555, + "candidate_abs_scaled_error": 0.00016163, + "candidate_error": -0.00016163, + "candidate_estimate": 0.0140317, + "candidate_loss_term": 2.61244e-08, + "candidate_relative_error": 0.985846, + "family": "state_agi_distribution", + "loss_delta": -0.140428, + "split": "train", + "target_index": 3581, + "target_name": "state/WV/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.369288, + "baseline_error": 296062.0, + "baseline_estimate": 312278.0, + "baseline_loss_term": 0.136374, + "baseline_relative_error": 18.2564, + "candidate_abs_scaled_error": 0.00438848, + "candidate_error": 3517.31, + "candidate_estimate": 19733.2, + "candidate_loss_term": 1.92588e-05, + "candidate_relative_error": 0.216953, + "family": "national_irs_other", + "loss_delta": -0.136355, + "split": "holdout", + "target_index": 386, + "target_name": "nation/irs/partnership and s corp losses/count/AGI in 20k-25k/taxable/All", + "target_value": 16215.9, + "value_scale": "native", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.282962, + "baseline_error": 0.282962, + "baseline_estimate": 0.30319, + "baseline_loss_term": 0.0800674, + "baseline_relative_error": 1.25752, + "candidate_abs_scaled_error": 0.0202278, + "candidate_error": -0.0202278, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000409165, + "candidate_relative_error": 0.960346, + "family": "national_irs_other", + "loss_delta": -0.0796583, + "split": "train", + "target_index": 31, + "target_name": "nation/irs/adjusted gross income/total/AGI in 20k-25k/taxable/Married Filing Jointly/Surviving Spouse", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.257691, + "baseline_error": 0.257691, + "baseline_estimate": 0.277918, + "baseline_loss_term": 0.0664045, + "baseline_relative_error": 1.23275, + "candidate_abs_scaled_error": 0.00211706, + "candidate_error": -0.00211706, + "candidate_estimate": 0.0181108, + "candidate_loss_term": 4.48196e-06, + "candidate_relative_error": 0.978098, + "family": "national_tax_expenditures", + "loss_delta": -0.0664, + "split": "train", + "target_index": 1475, + "target_name": "nation/jct/charitable_deduction_expenditure", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.242715, + "baseline_error": 0.242715, + "baseline_estimate": 0.256909, + "baseline_loss_term": 0.0589107, + "baseline_relative_error": 1.22532, + "candidate_abs_scaled_error": 0.00130261, + "candidate_error": 0.00130261, + "candidate_estimate": 0.0154959, + "candidate_loss_term": 1.6968e-06, + "candidate_relative_error": 0.98729, + "family": "state_agi_distribution", + "loss_delta": -0.058909, + "split": "train", + "target_index": 3112, + "target_name": "state/AK/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.239979, + "baseline_error": 0.239979, + "baseline_estimate": 0.254172, + "baseline_loss_term": 0.0575899, + "baseline_relative_error": 1.22263, + "candidate_abs_scaled_error": 0.00272689, + "candidate_error": 0.00272689, + "candidate_estimate": 0.0169202, + "candidate_loss_term": 7.4359e-06, + "candidate_relative_error": 0.988694, + "family": "state_agi_distribution", + "loss_delta": -0.0575825, + "split": "train", + "target_index": 3242, + "target_name": "state/IN/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.228938, + "baseline_error": 0.228938, + "baseline_estimate": 0.243132, + "baseline_loss_term": 0.0524128, + "baseline_relative_error": 1.21174, + "candidate_abs_scaled_error": 0.00722387, + "candidate_error": -0.00722387, + "candidate_estimate": 0.00696945, + "candidate_loss_term": 5.21843e-05, + "candidate_relative_error": 0.978883, + "family": "state_agi_distribution", + "loss_delta": -0.0523606, + "split": "holdout", + "target_index": 3292, + "target_name": "state/ME/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.217043, + "baseline_error": 16614.7, + "baseline_estimate": 18162.3, + "baseline_loss_term": 0.0471076, + "baseline_relative_error": 10.7299, + "candidate_abs_scaled_error": 0.00430105, + "candidate_error": 328.267, + "candidate_estimate": 1875.81, + "candidate_loss_term": 1.8499e-05, + "candidate_relative_error": 0.21263, + "family": "national_irs_other", + "loss_delta": -0.0470891, + "split": "train", + "target_index": 575, + "target_name": "nation/irs/count/count/AGI in 79m-inf/taxable/All", + "target_value": 1547.54, + "value_scale": "native", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.216355, + "baseline_error": 0.216355, + "baseline_estimate": 0.230548, + "baseline_loss_term": 0.0468095, + "baseline_relative_error": 1.19933, + "candidate_abs_scaled_error": 0.000426139, + "candidate_error": 0.000426139, + "candidate_estimate": 0.0146195, + "candidate_loss_term": 1.81594e-07, + "candidate_relative_error": 0.986425, + "family": "state_agi_distribution", + "loss_delta": -0.0468093, + "split": "train", + "target_index": 3352, + "target_name": "state/MO/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.203653, + "baseline_error": 11203.5, + "baseline_estimate": 12315.4, + "baseline_loss_term": 0.0414744, + "baseline_relative_error": 10.0679, + "candidate_abs_scaled_error": 0.0201915, + "candidate_error": -1111.89, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000407696, + "candidate_relative_error": -0.998203, + "family": "national_irs_other", + "loss_delta": -0.0410667, + "split": "train", + "target_index": 227, + "target_name": "nation/irs/count/count/AGI in 20k-25k/taxable/Married Filing Jointly/Surviving Spouse", + "target_value": 1111.89, + "value_scale": "native", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.192106, + "baseline_error": 0.192106, + "baseline_estimate": 0.212334, + "baseline_loss_term": 0.0369047, + "baseline_relative_error": 1.16847, + "candidate_abs_scaled_error": 0.00147261, + "candidate_error": 0.00147261, + "candidate_estimate": 0.0217004, + "candidate_loss_term": 2.16859e-06, + "candidate_relative_error": 0.981617, + "family": "national_irs_other", + "loss_delta": -0.0369025, + "split": "train", + "target_index": 65, + "target_name": "nation/irs/adjusted gross income/total/AGI in 1m-inf/taxable/Single", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.176761, + "baseline_error": 0.176761, + "baseline_estimate": 0.196989, + "baseline_loss_term": 0.0312445, + "baseline_relative_error": 1.15343, + "candidate_abs_scaled_error": 0.000669666, + "candidate_error": 0.000669666, + "candidate_estimate": 0.0208975, + "candidate_loss_term": 4.48453e-07, + "candidate_relative_error": 0.98083, + "family": "national_irs_other", + "loss_delta": -0.0312441, + "split": "train", + "target_index": 18, + "target_name": "nation/irs/adjusted gross income/total/AGI in 10m-inf/taxable/All", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.163597, + "baseline_error": 0.163597, + "baseline_estimate": 0.17779, + "baseline_loss_term": 0.0267639, + "baseline_relative_error": 1.14731, + "candidate_abs_scaled_error": 0.00286283, + "candidate_error": 0.00286283, + "candidate_estimate": 0.0170562, + "candidate_loss_term": 8.19582e-06, + "candidate_relative_error": 0.988828, + "family": "state_agi_distribution", + "loss_delta": -0.0267557, + "split": "holdout", + "target_index": 3511, + "target_name": "state/SD/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.163348, + "baseline_error": 0.163348, + "baseline_estimate": 0.177541, + "baseline_loss_term": 0.0266826, + "baseline_relative_error": 1.14707, + "candidate_abs_scaled_error": 0.00034419, + "candidate_error": -0.00034419, + "candidate_estimate": 0.0138491, + "candidate_loss_term": 1.18467e-07, + "candidate_relative_error": 0.985666, + "family": "state_agi_distribution", + "loss_delta": -0.0266825, + "split": "holdout", + "target_index": 3212, + "target_name": "state/HI/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.152645, + "baseline_error": 0.152645, + "baseline_estimate": 0.172873, + "baseline_loss_term": 0.0233006, + "baseline_relative_error": 1.12979, + "candidate_abs_scaled_error": 0.000889584, + "candidate_error": -0.000889584, + "candidate_estimate": 0.0193383, + "candidate_loss_term": 7.9136e-07, + "candidate_relative_error": 0.979301, + "family": "national_irs_other", + "loss_delta": -0.0232998, + "split": "train", + "target_index": 579, + "target_name": "nation/irs/partnership and s corp income/total/AGI in 79m-inf/taxable/All", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.144399, + "baseline_error": 0.144399, + "baseline_estimate": 0.158593, + "baseline_loss_term": 0.0208512, + "baseline_relative_error": 1.12838, + "candidate_abs_scaled_error": 0.00123666, + "candidate_error": 0.00123666, + "candidate_estimate": 0.01543, + "candidate_loss_term": 1.52934e-06, + "candidate_relative_error": 0.987225, + "family": "state_agi_distribution", + "loss_delta": -0.0208497, + "split": "holdout", + "target_index": 3272, + "target_name": "state/KY/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.127683, + "baseline_error": 0.127683, + "baseline_estimate": 0.147911, + "baseline_loss_term": 0.016303, + "baseline_relative_error": 1.10532, + "candidate_abs_scaled_error": 0.000284515, + "candidate_error": 0.000284515, + "candidate_estimate": 0.0205123, + "candidate_loss_term": 8.09485e-08, + "candidate_relative_error": 0.980452, + "family": "national_irs_other", + "loss_delta": -0.0163029, + "split": "train", + "target_index": 288, + "target_name": "nation/irs/estate income/total/AGI in -inf-inf/taxable/All", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.113918, + "baseline_error": 0.113918, + "baseline_estimate": 0.128112, + "baseline_loss_term": 0.0129774, + "baseline_relative_error": 1.09833, + "candidate_abs_scaled_error": 7.07679e-05, + "candidate_error": 7.07679e-05, + "candidate_estimate": 0.0142641, + "candidate_loss_term": 5.00809e-09, + "candidate_relative_error": 0.986075, + "family": "state_agi_distribution", + "loss_delta": -0.0129774, + "split": "train", + "target_index": 3392, + "target_name": "state/NH/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.108285, + "baseline_error": 0.108285, + "baseline_estimate": 0.122478, + "baseline_loss_term": 0.0117257, + "baseline_relative_error": 1.09277, + "candidate_abs_scaled_error": 0.00480168, + "candidate_error": 0.00480168, + "candidate_estimate": 0.018995, + "candidate_loss_term": 2.30562e-05, + "candidate_relative_error": 0.99074, + "family": "state_agi_distribution", + "loss_delta": -0.0117026, + "split": "train", + "target_index": 3102, + "target_name": "state/AL/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.111496, + "baseline_error": 0.111496, + "baseline_estimate": 0.125689, + "baseline_loss_term": 0.0124314, + "baseline_relative_error": 1.09594, + "candidate_abs_scaled_error": 0.0351581, + "candidate_error": 0.0351581, + "candidate_estimate": 0.0493515, + "candidate_loss_term": 0.00123609, + "candidate_relative_error": 1.02067, + "family": "state_agi_distribution", + "loss_delta": -0.0111953, + "split": "holdout", + "target_index": 3412, + "target_name": "state/NM/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.091256, + "baseline_error": 0.091256, + "baseline_estimate": 0.105449, + "baseline_loss_term": 0.00832766, + "baseline_relative_error": 1.07598, + "candidate_abs_scaled_error": 0.00358438, + "candidate_error": 0.00358438, + "candidate_estimate": 0.0177777, + "candidate_loss_term": 1.28478e-05, + "candidate_relative_error": 0.98954, + "family": "state_agi_distribution", + "loss_delta": -0.00831481, + "split": "train", + "target_index": 3521, + "target_name": "state/TN/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.0895466, + "baseline_error": 0.0895466, + "baseline_estimate": 0.109774, + "baseline_loss_term": 0.00801859, + "baseline_relative_error": 1.06794, + "candidate_abs_scaled_error": 0.00333663, + "candidate_error": 0.00333663, + "candidate_estimate": 0.0235645, + "candidate_loss_term": 1.11331e-05, + "candidate_relative_error": 0.983444, + "family": "national_irs_other", + "loss_delta": -0.00800746, + "split": "train", + "target_index": 584, + "target_name": "nation/irs/partnership and s corp income/total/AGI in 16m-79m/taxable/All", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.0862524, + "baseline_error": 0.0862524, + "baseline_estimate": 0.100446, + "baseline_loss_term": 0.00743948, + "baseline_relative_error": 1.07105, + "candidate_abs_scaled_error": 0.00401256, + "candidate_error": -0.00401256, + "candidate_estimate": 0.0101808, + "candidate_loss_term": 1.61006e-05, + "candidate_relative_error": 0.982049, + "family": "state_agi_distribution", + "loss_delta": -0.00742338, + "split": "train", + "target_index": 3402, + "target_name": "state/NJ/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + }, + { + "baseline_abs_scaled_error": 0.0846137, + "baseline_error": 0.0846137, + "baseline_estimate": 0.098807, + "baseline_loss_term": 0.00715948, + "baseline_relative_error": 1.06943, + "candidate_abs_scaled_error": 0.000617412, + "candidate_error": -0.000617412, + "candidate_estimate": 0.0135759, + "candidate_loss_term": 3.81197e-07, + "candidate_relative_error": 0.985397, + "family": "state_agi_distribution", + "loss_delta": -0.0071591, + "split": "train", + "target_index": 3222, + "target_name": "state/ID/adjusted_gross_income/amount/1000000_inf", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "candidate" + } + ], + "top_regressions": [ + { + "baseline_abs_scaled_error": 0.000437906, + "baseline_error": 1544.76, + "baseline_estimate": 51644.8, + "baseline_loss_term": 1.91762e-07, + "baseline_relative_error": 0.0308529, + "candidate_abs_scaled_error": 0.124565, + "candidate_error": 439700.0, + "candidate_estimate": 489800.0, + "candidate_loss_term": 0.0155164, + "candidate_relative_error": 8.77629, + "family": "state_agi_distribution", + "loss_delta": 0.0155162, + "split": "train", + "target_index": 3003, + "target_name": "state/TN/adjusted_gross_income/count/-inf_1", + "target_value": 50100.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000647692, + "baseline_error": 2878.52, + "baseline_estimate": 65978.5, + "baseline_loss_term": 4.19505e-07, + "baseline_relative_error": 0.0456336, + "candidate_abs_scaled_error": 0.0779542, + "candidate_error": 346569.0, + "candidate_estimate": 409669.0, + "candidate_loss_term": 0.00607685, + "candidate_relative_error": 5.49231, + "family": "state_agi_distribution", + "loss_delta": 0.00607643, + "split": "train", + "target_index": 2883, + "target_name": "state/NJ/adjusted_gross_income/count/-inf_1", + "target_value": 63100.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000484119, + "baseline_error": -0.000484119, + "baseline_estimate": 0.0197437, + "baseline_loss_term": 2.34371e-07, + "baseline_relative_error": 0.979699, + "candidate_abs_scaled_error": 0.0736296, + "candidate_error": 0.0736296, + "candidate_estimate": 0.0938574, + "candidate_loss_term": 0.00542132, + "candidate_relative_error": 1.05234, + "family": "national_irs_other", + "loss_delta": 0.00542108, + "split": "train", + "target_index": 1516, + "target_name": "nation/irs/aca_ptc/amount/state_22", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00037467, + "baseline_error": 0.00037467, + "baseline_estimate": 0.0206025, + "baseline_loss_term": 1.40378e-07, + "baseline_relative_error": 0.98054, + "candidate_abs_scaled_error": 0.062599, + "candidate_error": 0.062599, + "candidate_estimate": 0.0828268, + "candidate_loss_term": 0.00391863, + "candidate_relative_error": 1.04153, + "family": "national_irs_other", + "loss_delta": 0.00391849, + "split": "train", + "target_index": 1540, + "target_name": "nation/irs/aca_ptc/amount/state_34", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000425219, + "baseline_error": 901.097, + "baseline_estimate": 31011.1, + "baseline_loss_term": 1.80811e-07, + "baseline_relative_error": 0.0299591, + "candidate_abs_scaled_error": 0.0487787, + "candidate_error": 103482.0, + "candidate_estimate": 133592.0, + "candidate_loss_term": 0.00237936, + "candidate_relative_error": 3.43673, + "family": "state_agi_distribution", + "loss_delta": 0.00237918, + "split": "train", + "target_index": 2753, + "target_name": "state/KY/adjusted_gross_income/count/-inf_1", + "target_value": 30110.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000340959, + "baseline_error": 828.039, + "baseline_estimate": 35338.0, + "baseline_loss_term": 1.16253e-07, + "baseline_relative_error": 0.0240225, + "candidate_abs_scaled_error": 0.0449943, + "candidate_error": 109402.0, + "candidate_estimate": 143912.0, + "candidate_loss_term": 0.00202449, + "candidate_relative_error": 3.1701, + "family": "state_agi_distribution", + "loss_delta": 0.00202437, + "split": "train", + "target_index": 2943, + "target_name": "state/OK/adjusted_gross_income/count/-inf_1", + "target_value": 34510.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.0201183, + "baseline_error": -1757420.0, + "baseline_estimate": 9562.22, + "baseline_loss_term": 0.000404748, + "baseline_relative_error": -0.994587, + "candidate_abs_scaled_error": 0.0400306, + "candidate_error": 3496830.0, + "candidate_estimate": 5263820.0, + "candidate_loss_term": 0.00160245, + "candidate_relative_error": 1.97898, + "family": "national_population_by_age", + "loss_delta": 0.0011977, + "split": "train", + "target_index": 716, + "target_name": "nation/census/population_by_age/80", + "target_value": 1766980.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000889904, + "baseline_error": -0.000889904, + "baseline_estimate": 0.0193379, + "baseline_loss_term": 7.91929e-07, + "baseline_relative_error": 0.979301, + "candidate_abs_scaled_error": 0.0336636, + "candidate_error": 0.0336636, + "candidate_estimate": 0.0538915, + "candidate_loss_term": 0.00113324, + "candidate_relative_error": 1.01317, + "family": "national_irs_other", + "loss_delta": 0.00113245, + "split": "train", + "target_index": 1502, + "target_name": "nation/irs/aca_ptc/amount/state_15", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00154789, + "baseline_error": 18179.6, + "baseline_estimate": 255762.0, + "baseline_loss_term": 2.39597e-06, + "baseline_relative_error": 0.0765229, + "candidate_abs_scaled_error": 0.0301692, + "candidate_error": 354348.0, + "candidate_estimate": 591931.0, + "candidate_loss_term": 0.000910182, + "candidate_relative_error": 1.49147, + "family": "national_irs_other", + "loss_delta": 0.000907786, + "split": "train", + "target_index": 1539, + "target_name": "nation/irs/aca_ptc/returns/state_34", + "target_value": 237583.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00037691, + "baseline_error": 4450.78, + "baseline_estimate": 172091.0, + "baseline_loss_term": 1.42061e-07, + "baseline_relative_error": 0.0265554, + "candidate_abs_scaled_error": 0.0270416, + "candidate_error": 319394.0, + "candidate_estimate": 487034.0, + "candidate_loss_term": 0.000731247, + "candidate_relative_error": 1.90523, + "family": "state_agi_distribution", + "loss_delta": 0.000731105, + "split": "train", + "target_index": 2903, + "target_name": "state/NY/adjusted_gross_income/count/-inf_1", + "target_value": 167640.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00157985, + "baseline_error": -0.00157985, + "baseline_estimate": 0.0126135, + "baseline_loss_term": 2.49592e-06, + "baseline_relative_error": 0.984448, + "candidate_abs_scaled_error": 0.026557, + "candidate_error": 0.026557, + "candidate_estimate": 0.0407503, + "candidate_loss_term": 0.000705273, + "candidate_relative_error": 1.01219, + "family": "other", + "loss_delta": 0.000702777, + "split": "train", + "target_index": 1155, + "target_name": "state/irs/real_estate_taxes/KY", + "target_value": 0.0141933, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000467329, + "baseline_error": 1664.43, + "baseline_estimate": 52244.4, + "baseline_loss_term": 2.18396e-07, + "baseline_relative_error": 0.0329259, + "candidate_abs_scaled_error": 0.0224291, + "candidate_error": 79930.1, + "candidate_estimate": 130510.0, + "candidate_loss_term": 0.000503066, + "candidate_relative_error": 1.58026, + "family": "state_agi_distribution", + "loss_delta": 0.000502848, + "split": "holdout", + "target_index": 3043, + "target_name": "state/VA/adjusted_gross_income/count/-inf_1", + "target_value": 50580.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000163207, + "baseline_error": 375.944, + "baseline_estimate": 33155.9, + "baseline_loss_term": 2.66365e-08, + "baseline_relative_error": 0.0114989, + "candidate_abs_scaled_error": 0.021712, + "candidate_error": 50145.2, + "candidate_estimate": 82925.2, + "candidate_loss_term": 0.000471411, + "candidate_relative_error": 1.52973, + "family": "state_agi_distribution", + "loss_delta": 0.000471384, + "split": "train", + "target_index": 2953, + "target_name": "state/OR/adjusted_gross_income/count/-inf_1", + "target_value": 32780.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00340543, + "baseline_error": 698.394, + "baseline_estimate": 4851.71, + "baseline_loss_term": 1.15969e-05, + "baseline_relative_error": 0.168354, + "candidate_abs_scaled_error": 0.0212041, + "candidate_error": 4353.83, + "candidate_estimate": 8507.15, + "candidate_loss_term": 0.000449615, + "candidate_relative_error": 1.04827, + "family": "national_irs_other", + "loss_delta": 0.000438018, + "split": "train", + "target_index": 943, + "target_name": "nation/irs/eitc/returns/c0_25k_30k", + "target_value": 4153.32, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000650345, + "baseline_error": 0.000650345, + "baseline_estimate": 0.0208782, + "baseline_loss_term": 4.22948e-07, + "baseline_relative_error": 0.980811, + "candidate_abs_scaled_error": 0.0202278, + "candidate_error": -0.0202278, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000409165, + "candidate_relative_error": 0.960346, + "family": "national_ssa", + "loss_delta": 0.000408742, + "split": "train", + "target_index": 1478, + "target_name": "nation/ssa/ssn_card_type_none_count", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000673063, + "baseline_error": -0.000673063, + "baseline_estimate": 0.0195548, + "baseline_loss_term": 4.53013e-07, + "baseline_relative_error": 0.979514, + "candidate_abs_scaled_error": 0.020213, + "candidate_error": -0.020213, + "candidate_estimate": 1.48429e-05, + "candidate_loss_term": 0.000408565, + "candidate_relative_error": 0.960361, + "family": "national_irs_other", + "loss_delta": 0.000408112, + "split": "train", + "target_index": 1010, + "target_name": "nation/irs/eitc/amount/c2_4k_5k", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00267497, + "baseline_error": 1771.29, + "baseline_estimate": 15172.2, + "baseline_loss_term": 7.15544e-06, + "baseline_relative_error": 0.132242, + "candidate_abs_scaled_error": 0.0203646, + "candidate_error": 13491.5, + "candidate_estimate": 26892.4, + "candidate_loss_term": 0.000414717, + "candidate_relative_error": 1.00676, + "family": "national_irs_other", + "loss_delta": 0.000407561, + "split": "train", + "target_index": 1501, + "target_name": "nation/irs/aca_ptc/returns/state_15", + "target_value": 13400.9, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00144826, + "baseline_error": 82088.0, + "baseline_estimate": 1228630.0, + "baseline_loss_term": 2.09745e-06, + "baseline_relative_error": 0.0715972, + "candidate_abs_scaled_error": 0.0202278, + "candidate_error": -1146540.0, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000409164, + "candidate_relative_error": -0.999998, + "family": "national_population_by_age", + "loss_delta": 0.000407066, + "split": "train", + "target_index": 720, + "target_name": "nation/census/population_by_age/84", + "target_value": 1146540.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00207857, + "baseline_error": 150748.0, + "baseline_estimate": 1617770.0, + "baseline_loss_term": 4.32045e-06, + "baseline_relative_error": 0.102758, + "candidate_abs_scaled_error": 0.0202278, + "candidate_error": -1467030.0, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000409164, + "candidate_relative_error": -0.999999, + "family": "national_population_by_age", + "loss_delta": 0.000404844, + "split": "train", + "target_index": 718, + "target_name": "nation/census/population_by_age/82", + "target_value": 1467030.0, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00225194, + "baseline_error": 0.00225194, + "baseline_estimate": 0.0224798, + "baseline_loss_term": 5.07123e-06, + "baseline_relative_error": 0.982381, + "candidate_abs_scaled_error": 0.0202278, + "candidate_error": -0.0202278, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000409165, + "candidate_relative_error": 0.960346, + "family": "national_irs_other", + "loss_delta": 0.000404094, + "split": "holdout", + "target_index": 51, + "target_name": "nation/irs/adjusted gross income/total/AGI in 500k-1m/taxable/Married Filing Separately", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00204695, + "baseline_error": 4512.17, + "baseline_estimate": 49110.1, + "baseline_loss_term": 4.18999e-06, + "baseline_relative_error": 0.101195, + "candidate_abs_scaled_error": 0.0202034, + "candidate_error": -44546.0, + "candidate_estimate": 51.928, + "candidate_loss_term": 0.000408176, + "candidate_relative_error": -0.998791, + "family": "national_irs_other", + "loss_delta": 0.000403986, + "split": "train", + "target_index": 1009, + "target_name": "nation/irs/eitc/returns/c2_4k_5k", + "target_value": 44597.9, + "value_scale": "native", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000126119, + "baseline_error": -0.000126119, + "baseline_estimate": 0.0201017, + "baseline_loss_term": 1.5906e-08, + "baseline_relative_error": 0.98005, + "candidate_abs_scaled_error": 0.0200973, + "candidate_error": -0.0200973, + "candidate_estimate": 0.000130562, + "candidate_loss_term": 0.0004039, + "candidate_relative_error": 0.960474, + "family": "national_irs_other", + "loss_delta": 0.000403884, + "split": "holdout", + "target_index": 1002, + "target_name": "nation/irs/eitc/amount/c2_1_1k", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 1.84668e-05, + "baseline_error": -1.84668e-05, + "baseline_estimate": 0.0202094, + "baseline_loss_term": 3.41024e-10, + "baseline_relative_error": 0.980155, + "candidate_abs_scaled_error": 0.0200876, + "candidate_error": -0.0200876, + "candidate_estimate": 0.000140237, + "candidate_loss_term": 0.000403512, + "candidate_relative_error": 0.960484, + "family": "national_irs_other", + "loss_delta": 0.000403511, + "split": "train", + "target_index": 1016, + "target_name": "nation/irs/eitc/amount/c2_7k_8k", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.00242614, + "baseline_error": 0.00242614, + "baseline_estimate": 0.022654, + "baseline_loss_term": 5.88614e-06, + "baseline_relative_error": 0.982551, + "candidate_abs_scaled_error": 0.0202278, + "candidate_error": -0.0202278, + "candidate_estimate": 0.0, + "candidate_loss_term": 0.000409165, + "candidate_relative_error": 0.960346, + "family": "national_irs_other", + "loss_delta": 0.000403279, + "split": "train", + "target_index": 377, + "target_name": "nation/irs/partnership and s corp losses/total/AGI in -inf-0/taxable/All", + "target_value": 0.0202278, + "value_scale": "scaled", + "winner": "baseline" + }, + { + "baseline_abs_scaled_error": 0.000695764, + "baseline_error": 1541.91, + "baseline_estimate": 46397.8, + "baseline_loss_term": 4.84088e-07, + "baseline_relative_error": 0.0343964, + "candidate_abs_scaled_error": 0.0200556, + "candidate_error": -44475.9, + "candidate_estimate": 379.968, + "candidate_loss_term": 0.000402227, + "candidate_relative_error": -0.991485, + "family": "national_irs_other", + "loss_delta": 0.000401743, + "split": "train", + "target_index": 1015, + "target_name": "nation/irs/eitc/returns/c2_7k_8k", + "target_value": 44855.9, + "value_scale": "native", + "winner": "baseline" + } + ], + "gates": { + "ecps_refit_recovery": "pass" + } +} \ No newline at end of file diff --git a/benchmarks/us/incumbent-comparison/README.md b/benchmarks/us/incumbent-comparison/README.md index 666a42a..565b142 100644 --- a/benchmarks/us/incumbent-comparison/README.md +++ b/benchmarks/us/incumbent-comparison/README.md @@ -37,3 +37,35 @@ Use only the pinned production incumbent recorded in `manifests/pinned-production-ecps-2024.json` for promotion comparisons. Known local or development copies of the incumbent are not equivalent, even when they share a similar file name. + +## Published Scorecards + +Each comparison run publishes a machine-readable scorecard so consumers (the +[calibration-diagnostics dashboard](https://github.com/PolicyEngine/calibration-diagnostics)) +can read the head-to-head without re-running the harness: + +- **Scorecard** — `archive/us//scorecard.json`, conforming + to [`scorecard.schema.json`](scorecard.schema.json). It carries the promotion + metrics (full / holdout loss, unweighted MSRE), per-target win/loss/tie, + per-family loss breakdown, and the top movers. +- **Pointer** — [`latest.json`](latest.json) names the most recent scorecard + (`scorecard_path` + `candidate_release_id`), so a consumer resolves the + current scorecard without hard-coding a path. + +A scorecard's `status` is `archived` when it was recomputed from a historical +candidate's published comparison (rather than a fresh benchmark run against the +pinned incumbent HDF5s). The first published scorecard, +`populace-us-2024-9f1260b-20260611`, is archived: it was reconstructed from that +release's `sound_ecps_replacement_comparison.json` on the populace-us dataset. + +### Validation + +`tools/validate_scorecard.py` (stdlib only) checks every scorecard and pointer — +required keys, promotion-metric types, and internal consistency the schema +cannot express (win counts sum to the target count; `candidate_beats_baseline` +agrees with the loss values). It runs in CI on every PR +(`.github/workflows/validate-scorecards.yml`) and locally: + +```bash +python tools/validate_scorecard.py +``` diff --git a/benchmarks/us/incumbent-comparison/latest.json b/benchmarks/us/incumbent-comparison/latest.json new file mode 100644 index 0000000..727894f --- /dev/null +++ b/benchmarks/us/incumbent-comparison/latest.json @@ -0,0 +1,9 @@ +{ + "schema_version": 1, + "candidate_release_id": "populace-us-2024-9f1260b-20260611", + "incumbent_manifest": "pinned-production-ecps-2024", + "scorecard_path": "archive/us/populace-us-2024-9f1260b-20260611/scorecard.json", + "updated_at": "2026-06-15T00:00:00+00:00", + "status": "archived", + "note": "Most recent published incumbent-comparison scorecard. 'archived' marks a scorecard recomputed from a historical candidate's published comparison rather than a fresh benchmark run." +} diff --git a/benchmarks/us/incumbent-comparison/scorecard.schema.json b/benchmarks/us/incumbent-comparison/scorecard.schema.json new file mode 100644 index 0000000..bbe172b --- /dev/null +++ b/benchmarks/us/incumbent-comparison/scorecard.schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/PolicyEngine/populace-benchmarks/benchmarks/us/incumbent-comparison/scorecard.schema.json", + "title": "US incumbent-comparison scorecard", + "description": "Machine-readable result of scoring a candidate Populace-US artifact against the certified pinned production incumbent on the frozen target surface. One scorecard per benchmark run; consumers (e.g. the calibration-diagnostics dashboard) read it read-only.", + "type": "object", + "required": [ + "schema_version", + "candidate_release_id", + "incumbent_manifest", + "baseline_label", + "candidate_label", + "summary" + ], + "additionalProperties": true, + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "candidate_release_id": { "type": "string", "minLength": 1 }, + "incumbent_manifest": { "type": "string", "minLength": 1 }, + "period": { "type": ["integer", "null"] }, + "metric": { "type": ["string", "null"] }, + "baseline_label": { "type": "string" }, + "candidate_label": { "type": "string" }, + "protocol": { "type": ["string", "null"] }, + "provenance": { "type": "object" }, + "summary": { + "type": "object", + "description": "Flat head-to-head summary. The promotion metrics (full loss, holdout loss, unweighted MSRE) are required; candidate must beat the incumbent on each to be promotion-valid.", + "required": [ + "candidate_loss", + "baseline_loss", + "candidate_holdout_loss", + "baseline_holdout_loss", + "candidate_unweighted_msre", + "baseline_unweighted_msre", + "candidate_wins", + "baseline_wins", + "ties", + "n_targets" + ], + "additionalProperties": true, + "properties": { + "candidate_loss": { "type": "number" }, + "baseline_loss": { "type": "number" }, + "loss_delta": { "type": ["number", "null"] }, + "candidate_holdout_loss": { "type": "number" }, + "baseline_holdout_loss": { "type": "number" }, + "candidate_train_loss": { "type": ["number", "null"] }, + "baseline_train_loss": { "type": ["number", "null"] }, + "candidate_unweighted_msre": { "type": "number" }, + "baseline_unweighted_msre": { "type": "number" }, + "candidate_wins": { "type": "integer", "minimum": 0 }, + "baseline_wins": { "type": "integer", "minimum": 0 }, + "ties": { "type": "integer", "minimum": 0 }, + "n_targets": { "type": "integer", "minimum": 1 }, + "holdout_targets": { "type": ["integer", "null"] }, + "train_targets": { "type": ["integer", "null"] }, + "candidate_beats_baseline": { "type": ["boolean", "null"] }, + "matched_household_count": { "type": ["integer", "null"] } + } + }, + "family_breakdown": { + "type": "array", + "items": { + "type": "object", + "required": ["family", "n_targets"], + "additionalProperties": true, + "properties": { + "family": { "type": "string" }, + "n_targets": { "type": "integer", "minimum": 0 }, + "candidate_wins": { "type": ["integer", "null"] }, + "baseline_wins": { "type": ["integer", "null"] }, + "ties": { "type": ["integer", "null"] }, + "candidate_loss_contribution": { "type": ["number", "null"] }, + "baseline_loss_contribution": { "type": ["number", "null"] }, + "loss_delta": { "type": ["number", "null"] } + } + } + }, + "top_improvements": { "type": "array" }, + "top_regressions": { "type": "array" }, + "gates": { "type": "object" } + } +} diff --git a/tools/validate_scorecard.py b/tools/validate_scorecard.py new file mode 100644 index 0000000..a8f2511 --- /dev/null +++ b/tools/validate_scorecard.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Validate published incumbent-comparison scorecards (stdlib only). + +A scorecard is the machine-readable result a candidate-vs-incumbent benchmark +run publishes (see ``benchmarks/us/incumbent-comparison/scorecard.schema.json``). +Consumers read it read-only, so a malformed or internally-inconsistent scorecard +is a published-contract break. This validator is the gate: it runs in CI on every +PR and is runnable locally with no dependencies. + +It checks each scorecard for required keys and types, the promotion metrics, and +internal consistency the JSON Schema cannot express (win counts sum to the target +count). It also checks that every ``latest.json`` pointer resolves to a scorecard +that exists and whose ``candidate_release_id`` matches the pointer. + +Usage: + python tools/validate_scorecard.py # validate everything tracked + python tools/validate_scorecard.py PATH ... # validate specific files +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +REQUIRED_TOP = ( + "schema_version", + "candidate_release_id", + "incumbent_manifest", + "baseline_label", + "candidate_label", + "summary", +) +# Promotion metrics (the benchmark manifest's required set) plus the win/tie +# counts the consistency check needs. +REQUIRED_SUMMARY_NUMERIC = ( + "candidate_loss", + "baseline_loss", + "candidate_holdout_loss", + "baseline_holdout_loss", + "candidate_unweighted_msre", + "baseline_unweighted_msre", +) +REQUIRED_SUMMARY_COUNTS = ("candidate_wins", "baseline_wins", "ties", "n_targets") + + +def _is_number(value: object) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def validate_scorecard(path: Path) -> list[str]: + errors: list[str] = [] + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + return [f"{path}: invalid JSON: {exc}"] + if not isinstance(data, dict): + return [f"{path}: top level must be a JSON object"] + + for key in REQUIRED_TOP: + if key not in data: + errors.append(f"{path}: missing required key {key!r}") + + if data.get("schema_version") != 1: + errors.append( + f"{path}: schema_version must be 1, got {data.get('schema_version')!r}" + ) + + summary = data.get("summary") + if not isinstance(summary, dict): + errors.append(f"{path}: 'summary' must be an object") + return errors + + for key in REQUIRED_SUMMARY_NUMERIC: + if not _is_number(summary.get(key)): + errors.append(f"{path}: summary.{key} must be a number") + for key in REQUIRED_SUMMARY_COUNTS: + value = summary.get(key) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + errors.append(f"{path}: summary.{key} must be a non-negative integer") + + # Internal consistency the schema cannot express: the per-target verdicts + # partition the scored targets exactly. + counts = [summary.get(k) for k in ("candidate_wins", "baseline_wins", "ties")] + n_targets = summary.get("n_targets") + if all(isinstance(c, int) and not isinstance(c, bool) for c in counts) and isinstance( + n_targets, int + ): + total = sum(counts) + if total != n_targets: + errors.append( + f"{path}: candidate_wins + baseline_wins + ties = {total} " + f"!= n_targets = {n_targets}" + ) + + # candidate_beats_baseline, when present, must agree with the loss sign. + beats = summary.get("candidate_beats_baseline") + if isinstance(beats, bool) and _is_number(summary.get("candidate_loss")) and _is_number( + summary.get("baseline_loss") + ): + actual = summary["candidate_loss"] < summary["baseline_loss"] + if beats != actual: + errors.append( + f"{path}: candidate_beats_baseline={beats} disagrees with the " + f"loss values ({summary['candidate_loss']} vs " + f"{summary['baseline_loss']})" + ) + + return errors + + +def validate_pointer(path: Path) -> list[str]: + errors: list[str] = [] + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + return [f"{path}: invalid JSON: {exc}"] + + scorecard_path = data.get("scorecard_path") + if not isinstance(scorecard_path, str) or not scorecard_path: + return [f"{path}: pointer missing 'scorecard_path'"] + target = REPO_ROOT / scorecard_path + if not target.is_file(): + return [f"{path}: scorecard_path {scorecard_path!r} does not exist"] + + errors.extend(validate_scorecard(target)) + pointer_release = data.get("candidate_release_id") + scorecard_release = json.loads(target.read_text()).get("candidate_release_id") + if pointer_release != scorecard_release: + errors.append( + f"{path}: pointer candidate_release_id {pointer_release!r} != " + f"scorecard {scorecard_release!r}" + ) + return errors + + +def discover() -> tuple[list[Path], list[Path]]: + scorecards = sorted((REPO_ROOT / "archive").rglob("scorecard.json")) + pointers = sorted((REPO_ROOT / "benchmarks").rglob("latest.json")) + return scorecards, pointers + + +def main(argv: list[str]) -> int: + if argv: + scorecards = [Path(a) for a in argv if Path(a).name != "latest.json"] + pointers = [Path(a) for a in argv if Path(a).name == "latest.json"] + else: + scorecards, pointers = discover() + + errors: list[str] = [] + for path in scorecards: + errors.extend(validate_scorecard(path)) + for path in pointers: + errors.extend(validate_pointer(path)) + + checked = len(scorecards) + len(pointers) + if errors: + print(f"FAIL: {len(errors)} problem(s) across {checked} file(s):") + for error in errors: + print(f" - {error}") + return 1 + print(f"OK: {checked} file(s) valid ({len(scorecards)} scorecard(s), " + f"{len(pointers)} pointer(s)).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:]))