From 8262194cd7b3eb577442778067d74efb0b56b120 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 13 Jul 2026 04:30:14 -0400 Subject: [PATCH] Chesterton's-fence audit: wire NIPA wages/proprietors, net worth, LIHEAP; fence every exclusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the rule us-data-targeted => compiled (unless source-absent) to every target the retired us-data/eCPS pipeline calibrated to, recovered by PR archaeology on the archived policyengine-us-data repo. Wired (were reviewed exclusions, now compiled), each following us-data's exact concept mapping: - bea_nipa.total_wages_salaries -> employment_income_before_lsr, national ($12,387,929,000,000, exact). BEA NIPA all-population wages incl. nonfilers (loss.py BEA_NIPA_DIRECT_SUM_TARGETS, PR #994) — the ~$12.4T universe that filed-return / CPS wages undercount by two-thirds. Added to the gate red line. - bea_nipa.proprietors_income -> self_employment_income_before_lsr + sstb_self_employment_income_before_lsr + farm_operations_income + partnership_s_corp_income, national (PR #994). - federal_reserve_z1 net worth -> net_worth, national (PR #282; linear direct sum pinning the post-reweight wealth aggregate). - hhs_acf_liheap.national_profile -> indicator sum of spm_unit_energy_subsidy (LIHEAP recipient households, PR #688) — the SNAP-caseload pattern. Fence schema (cannot-rot): every reviewed exclusion now carries a {origin, purpose, verdict_basis} recovering why the target existed and the purpose-informed basis for the exclusion; the regenerator refuses a fenceless exclusion. Bare category labels are no longer sufficient. Verdicts recorded: - bea interest/dividends: us-data itself declined (PR #994 added -> #1059 removed): imputed interest/pension-plan dividends/trust flows, not a close microdata concept. - bea_regional state wages: deferred (PR #1034) — the feed's raw place-of-work SAINC4 line-50 facts can't reproduce us-data's residence adjustment + national scaling; the national wage aggregate is compiled. - cms_nhe medicaid spending: architecture-retires (PR #292 -> PE-US #1138 made cost person_weight-dependent; validation_only). - retirement contributions: architecture-retires (PR #496/#554 -> #1125: governed as imputed input columns, not reweighting targets). - tip_income: deferred (PR #220) — structural zero until the tip source stage is ported. - the remaining BEA/KFF/census-projection/SNAP-persons/TANF-caseload/SOI-CD families carry a "not a us-data target" fence (verified absent from loss.py + etl_national_targets.py). Registry 5,689 -> 5,693 specs; manifest 25 -> 29 compiled, 55 -> 51 reviewed exclusions. Deterministic, Ruff clean. Wiring tests per family + the fence schema tests. Co-Authored-By: Claude Fable 5 --- .../build/us/target_parity_manifest.json | 488 +++++++++++---- .../src/populace/build/us_runtime/__init__.py | 6 +- .../build/us_runtime/fiscal_targets.py | 116 ++++ .../build/us_runtime/release_target_parity.py | 104 +++- .../populace/build/us_runtime/target_aging.py | 7 + .../tests/test_release_target_parity.py | 140 +++-- .../tests/test_us_fiscal_targets.py | 165 ++++++ tools/build_us_target_parity_manifest.py | 561 +++++++++++++----- 8 files changed, 1228 insertions(+), 359 deletions(-) diff --git a/packages/populace-build/src/populace/build/us/target_parity_manifest.json b/packages/populace-build/src/populace/build/us/target_parity_manifest.json index 656fbce6..6ae1d839 100644 --- a/packages/populace-build/src/populace/build/us/target_parity_manifest.json +++ b/packages/populace-build/src/populace/build/us/target_parity_manifest.json @@ -5,227 +5,392 @@ "feed_sha256": "94b7155f7ca9e2de32ddb3a0add2fff2d8c66e73147fe5bd112cff3ba69b1669", "target_period": "2024", "registry_compile": "compile_us_fiscal_target_registry(age_targets=True) + apply_us_medicaid_enrollment_substitutions", - "us_data_source": "retired us-data pipeline (archived): db/etl_national_targets.py, db/etl_*.py, utils/national_target_parity.py", + "us_data_source": "retired us-data pipeline (archived): utils/loss.py (eCPS loss matrix), db/etl_national_targets.py, db/etl_*.py, utils/national_target_parity.py", "family_granularity": "namespace.concept of the ledger source_record_id (us_target_family_id)", - "compiled_families": "25", - "reviewed_exclusions": "55" + "governing_rule": "us-data-targeted => compiled unless source-absent; every reviewed exclusion carries a Chesterton's-fence {origin, purpose, verdict_basis}", + "compiled_families": "29", + "reviewed_exclusions": "51" }, "families": { "bea_nipa.business_current_transfer_receipts": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.defined_contribution_actual_contributions": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.defined_contribution_employer_contributions": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.disposable_personal_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.employer_government_social_insurance_contributions": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.employer_pension_and_insurance_contributions": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.farm_proprietors_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.government_social_benefits_to_persons": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.medicaid_benefits": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.medicare_benefits": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.nonfarm_proprietors_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.other_government_social_benefits": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.personal_current_taxes": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.personal_current_transfer_receipts": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.personal_dividend_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA personal dividend income. Briefly a direct target (PR #994), then explicitly declined (PR #1059): the NIPA total includes dividends received through pension funds and private trusts.", + "evidence": "cbo.revenue_projection (qualified dividends) + irs_soi (compiled); sample fact bea_nipa.cy2023.personal_dividend_income.b703rc.amount", + "fence": { + "origin": "us-data PR #994 (added) -> PR #1059 (removed)", + "purpose": "\"NIPA includes dividends received through pension funds and private trusts, so this is a macro benchmark rather than a pure tax concept\" (deleted etl_national_targets.py note, PR #1059; FRED B703RC1A027NBEA)", + "verdict_basis": "us-data itself declined it (PR #1059) as non-comparable; dividends are calibrated from the CBO qualified-dividend series (cbo.revenue_projection) and SOI (compiled)." + } }, "bea_nipa.personal_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.personal_interest_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA personal interest income. Briefly a direct target (PR #994), then explicitly declined (PR #1059): the NIPA total includes imputed interest and trust flows that are not a close microdata concept.", + "evidence": "irs_soi.historic_table_2 (compiled) carries the tax-return interest; sample fact bea_nipa.cy2023.personal_interest_income.a064rc.amount", + "fence": { + "origin": "us-data PR #994 (added) -> PR #1059 (removed)", + "purpose": "\"BEA personal interest/dividends include imputed interest, pension-plan dividends, and trust flows, so those macro totals should not directly calibrate tax/CPS interest and dividend variables\" (loss.py:67-70, added by PR #1059)", + "verdict_basis": "us-data itself declined it as a direct target (PR #1059): a macro benchmark, not a close microdata concept. Tax-return interest is calibrated from IRS SOI + CBO (compiled)." + } }, "bea_nipa.personal_outlays": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.personal_saving": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.personal_saving_rate": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.proprietors_income": { - "status": "reviewed_exclusion", - "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "status": "compiled" }, "bea_nipa.rental_income_of_persons": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.social_security_benefits": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.supplements_to_wages_and_salaries": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.total_wages_salaries": { - "status": "reviewed_exclusion", - "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "status": "compiled" }, "bea_nipa.unemployment_insurance_benefits": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_nipa.veterans_benefits": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA NIPA national-accounts aggregate (calendar-year macro control total). Not a household-linear administrative level PolicyEngine-US calibrates against: household income components are fit from the IRS SOI micro tables (irs_soi.historic_table_2, compiled) and CBO revenue-by-source projections (cbo.revenue_projection, compiled). The NIPA aggregate is retained in the ledger as a macro cross-check, not a calibration target.", - "evidence": "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection supply the household-linear income surface; sample fact bea_nipa.cy2023.personal_income.a065rc.amount" + "reason": "BEA NIPA national-accounts aggregate the retired pipeline did not calibrate to (only wages and proprietors' income were NIPA direct-sum targets, both compiled). A macro control total retained as a ledger cross-check.", + "evidence": "compiled BEA targets bea_nipa.total_wages_salaries and bea_nipa.proprietors_income carry the household-linear NIPA income; sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + "fence": { + "origin": "not a us-data calibration target: absent from us-data utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA direct-sum targets were nipa_wages_and_salaries and proprietors_income (PR #994), both now compiled.", + "purpose": "n/a \u2014 a ledger reference fact carried by the feed, never a target.", + "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + } }, "bea_regional.state_contributions_for_government_social_insurance": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_dividends_interest_rent": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_personal_current_transfer_receipts": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_personal_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_proprietors_income": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_residence_adjustment": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_supplements_to_wages_salaries": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "reason": "BEA regional (state) account aggregate us-data did not calibrate to (only state wages were a regional target, and it is deferred). State income is fit from IRS SOI state tables and Census STC, compiled.", + "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount", + "fence": { + "origin": "not a us-data calibration target: absent from loss.py and etl_national_targets.py (archaeology). The only targeted bea_regional series is state_wages_salaries (PR #1034), a separate deferred exclusion.", + "purpose": "n/a \u2014 a ledger reference fact, never a target.", + "verdict_basis": "no fence to rebuild; state income is calibrated from IRS SOI state tables + Census STC (compiled)." + } }, "bea_regional.state_wages_salaries": { "status": "reviewed_exclusion", - "classification": "macro_control_total", - "reason": "BEA regional (state) personal-income account aggregate. State income is calibrated from the IRS SOI state tables (irs_soi.state_2022 and the historic_table_2 state rows, compiled) and Census STC (census_stc.individual_income_tax_collections, compiled); the BEA regional total is a macro state control without a per-record model counterpart.", - "evidence": "compiled siblings irs_soi.state_2022 and census_stc.individual_income_tax_collections; sample fact bea_regional.cy2023.state_personal_income.us.amount" + "classification": "deferred", + "reason": "BEA regional state wages \u2014 a real us-data target (PR #1034), but us-data residence-adjusts SAINC4 line-50 place-of-work wages and scales them to the national total, an adjustment the feed's raw place-of-work facts cannot reproduce. The national wage aggregate is compiled.", + "evidence": "compiled national aggregate bea_nipa.total_wages_salaries; feed fact bea_regional.cy2023.state_wages_salaries.dc.amount = $96.8B is place-of-work, far above DC residence wages", + "fence": { + "origin": "us-data PR #1034 (Fixes #1033)", + "purpose": "\"residence-adjust state wages using SAINC4 line 42 before scaling to the national NIPA wages target ... so the national and state wage controls bind\" (PR #1034); the state distribution of the nonfiler-inclusive wage universe", + "verdict_basis": "deferred: the feed carries raw place-of-work SAINC4 line-50 state wages, but us-data applies a place-of-work -> residence adjustment (line 42 apportioned by wage share) plus national scaling in bea_regional.py that is not reproducible from the feed. The national wage aggregate (bea_nipa.total_wages_salaries) is compiled; the state distribution is deferred pending residence-adjusted state wages in the ledger feed." + } }, "bls.consumer_expenditure": { "status": "reviewed_exclusion", "classification": "source_absent", - "reason": "BLS Consumer Expenditure Survey aggregates targeted by the retired us-data pipeline (nation/bls/ce). The pinned consumer feed carries no BLS source fact, so there is no ledger-shaped fact to compile; source-absent pending a Ledger BLS CE ingest.", - "evidence": "retired us-data pipeline (archived) references nation/bls/ce; feed source families carry no 'bls' source" + "reason": "BLS Consumer Expenditure childcare-expense target the retired pipeline calibrated (loss.py BLS_CE_TOTALS childcare_expenses = $63.09B). The pinned feed carries no BLS source fact.", + "evidence": "retired us-data pipeline (archived) loss.py BLS_CE_TOTALS + nation/bls/ce; feed source families carry no 'bls' source", + "fence": { + "origin": "us-data loss.py BLS_CE_TOTALS (BLS CE LABSTAT)", + "purpose": "anchor modeled childcare expenses to the BLS Consumer Expenditure aggregate ($63.09B, 2024).", + "verdict_basis": "source-absent: the pinned feed carries no BLS CE ledger fact, so there is nothing to compile; deferred pending a Ledger BLS CE ingest." + } }, "cbo.revenue_projection": { "status": "compiled" @@ -233,20 +398,35 @@ "cbo.revenues": { "status": "reviewed_exclusion", "classification": "macro_control_total", - "reason": "CBO total federal revenue line (fiscal-year budget aggregate). The household-linear CBO surface populace calibrates is the income-by-source projection (cbo.revenue_projection, compiled as cbo:5); a total revenue outturn is an aggregate, not a per-record target.", - "evidence": "compiled sibling cbo.revenue_projection; sample fact cbo.fy2023.revenues.individual_income_taxes.actual_amount" + "reason": "CBO total federal revenue line (fiscal-year budget aggregate) us-data did not calibrate to. The household-linear CBO surface it used is the income-by-source projection (cbo.revenue_projection, compiled).", + "evidence": "compiled sibling cbo.revenue_projection; sample fact cbo.fy2023.revenues.individual_income_taxes.actual_amount", + "fence": { + "origin": "not a us-data calibration target: us-data calibrates the CBO income-by-source projections (CBO_INCOME_BY_SOURCE_TARGETS), not the total-revenue outturn.", + "purpose": "n/a \u2014 an aggregate outturn, never a per-record target.", + "verdict_basis": "no fence to rebuild; component income is compiled via cbo.revenue_projection." + } }, "census.popproj2023": { "status": "reviewed_exclusion", "classification": "superseded", - "reason": "Census Bureau population PROJECTIONS (forward vintage popproj2023). Population is calibrated to the OBSERVED Census resident-population estimates (census_pep.v2024 and national_resident_population_age, compiled as census_pep:936); the projection series is a forward-looking alternative, not the observed administrative count.", - "evidence": "compiled sibling census_pep.v2024; sample fact census.popproj2023.cy2023.national_population.age_0.population" + "reason": "Census Bureau population PROJECTIONS (forward vintage). us-data calibrated the OBSERVED resident population, compiled via census_pep.", + "evidence": "compiled sibling census_pep.v2024; sample fact census.popproj2023.cy2023.national_population.age_0.population", + "fence": { + "origin": "not a us-data calibration target: us-data calibrates observed Census population (census_pep), not the projection series.", + "purpose": "n/a \u2014 a forward projection, never the observed control.", + "verdict_basis": "superseded by compiled census_pep.v2024 / national_resident_population_age (observed resident population)." + } }, "census_acs.acs1_2023": { "status": "reviewed_exclusion", "classification": "survey_derived", - "reason": "American Community Survey 1-year estimates. ACS is a household SAMPLE SURVEY, not an administrative universe, so its aggregates are excluded by principle. ACS age detail is used only at congressional-district grain when include_congressional_district_targets is enabled (off for the national release).", - "evidence": "census_acs facts return None unless include_congressional_district_targets in fiscal_targets._reference_from_ledger_fact; sample fact census_acs.acs1_2023.b01001.female_age.01.age_15_to_17.female_population" + "reason": "American Community Survey 1-year estimates \u2014 a household SAMPLE SURVEY, not an administrative universe. Used at congressional-district grain only, off for the national release.", + "evidence": "census_acs facts return None unless include_congressional_district_targets in fiscal_targets._reference_from_ledger_fact; sample fact census_acs.acs1_2023.b01001.female_age.01.age_15_to_17.female_population", + "fence": { + "origin": "not a national us-data calibration target: ACS feeds regional/CD H5 targets in us-data, off for the national build.", + "purpose": "n/a nationally \u2014 survey-derived CD detail, not a national fence.", + "verdict_basis": "off_by_default + survey_derived: excluded by principle (ACS is a sample survey) and only relevant at CD grain (include_congressional_district_targets, off here)." + } }, "census_pep.national_resident_population_age": { "status": "compiled" @@ -269,32 +449,41 @@ "cms_nhe.medicaid_title_xix_expenditures": { "status": "reviewed_exclusion", "classification": "non_linear", - "reason": "CMS National Health Expenditure Medicaid Title XIX spending. Already declared calibration_role=validation_only in DIRECT_LEDGER_TARGETS: PolicyEngine-US allocates Medicaid spending from state totals through person_weight-dependent denominators, so reweighting recomputes per-person costs and this is not a linear calibration row.", - "evidence": "DIRECT_LEDGER_TARGETS ('cms_nhe','expenditure_amount','medicaid_title_xix') metadata calibration_role=validation_only" + "reason": "CMS NHE Medicaid Title XIX spending \u2014 a real us-data target (PR #292) whose linear-target assumption PolicyEngine-US retired: medicaid cost is now a person_weight-dependent allocation, so this is not a linear calibration row. Kept as calibration_role=validation_only.", + "evidence": "DIRECT_LEDGER_TARGETS ('cms_nhe','expenditure_amount','medicaid_title_xix') metadata calibration_role=validation_only", + "fence": { + "origin": "us-data PR #292 (medicaid spending target)", + "purpose": "keep weighted Medicaid outlays aligned to the CMS national total (nation/hhs/medicaid_spending). No discoverable prose rationale in PR #292 (only a value/source note); the mechanical purpose is the aggregate-spending control.", + "verdict_basis": "architecture retires it: PolicyEngine-US PR #1138 made medicaid_cost_if_enrolled an SLCSP allocation from state totals normalized against enrollees \u2014 person_weight-dependent, so reweighting recomputes per-person costs and it is not a linear calibration row. populace keeps it calibration_role=validation_only in DIRECT_LEDGER_TARGETS." + } }, "federal_reserve_z1.households_nonprofits_balance_sheet": { - "status": "reviewed_exclusion", - "classification": "macro_control_total", - "reason": "Federal Reserve Financial Accounts (Z.1) households & nonprofits net-worth aggregate. A macro balance-sheet total; household net worth is imputed and calibrated from SCF micro on the input side (scf_wealth source stage), not from this national aggregate.", - "evidence": "scf_wealth.py source stage supplies the net-worth micro; sample fact federal_reserve_z1.cy2023.households_nonprofits_balance_sheet.net_worth.fl152090005.amount_outstanding" + "status": "compiled" }, "hhs_acf_liheap.national_profile": { - "status": "reviewed_exclusion", - "classification": "deferred", - "reason": "HHS LIHEAP national profile (households served / funds). PolicyEngine-US does not yet expose a LIHEAP receipt outcome to fit this count to a per-household model counterpart, so it is deferred rather than dropped; retained as a ledger reference fact.", - "evidence": "no LIHEAP target_role in INDICATOR_LEDGER_TARGETS/DIRECT_LEDGER_TARGETS; sample fact hhs_acf_liheap.fy2023.national_profile.state_programs.households_served" + "status": "compiled" }, "hhs_acf_tanf.average_monthly_families": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "HHS ACF TANF average-monthly family caseload. TANF is calibrated on BENEFIT DOLLARS (hhs_acf_tanf.cash_assistance, compiled as hhs_acf_tanf:30); the caseload COUNT is not yet wired to a TANF-receipt indicator (the SNAP-household caseload analog for TANF), so it is deferred.", - "evidence": "compiled sibling hhs_acf_tanf.cash_assistance carries the dollar targets; sample fact hhs_acf_tanf.fy2024.average_monthly_families.us.us_total.total_families" + "reason": "HHS ACF TANF average-monthly family caseload. TANF is calibrated on BENEFIT DOLLARS (compiled); the caseload COUNT has no wired TANF-receipt indicator yet.", + "evidence": "compiled sibling hhs_acf_tanf.cash_assistance; sample fact hhs_acf_tanf.fy2024.average_monthly_families.us.us_total.total_families", + "fence": { + "origin": "not a us-data calibration target: us-data calibrates TANF dollars (HARD_CODED_TOTALS \"tanf\"), compiled via cash_assistance; the caseload count is not a us-data target.", + "purpose": "n/a \u2014 TANF dollars are the target, not the caseload count.", + "verdict_basis": "deferred: no TANF-receipt indicator is wired (unlike the SNAP / LIHEAP household caseloads); the dollar target is compiled." + } }, "hhs_acf_tanf.average_monthly_recipients": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "HHS ACF TANF average-monthly recipient caseload. As with the family caseload, TANF is calibrated on benefit dollars (hhs_acf_tanf.cash_assistance, compiled); the recipient COUNT has no wired TANF-receipt indicator yet and is deferred.", - "evidence": "compiled sibling hhs_acf_tanf.cash_assistance; sample fact hhs_acf_tanf.fy2024.average_monthly_recipients.us.us_total.total_recipients" + "reason": "HHS ACF TANF average-monthly recipient caseload. As with families, TANF is calibrated on benefit dollars (compiled); the recipient COUNT has no wired TANF-receipt indicator.", + "evidence": "compiled sibling hhs_acf_tanf.cash_assistance; sample fact hhs_acf_tanf.fy2024.average_monthly_recipients.us.us_total.total_recipients", + "fence": { + "origin": "not a us-data calibration target: us-data calibrates TANF dollars (compiled); the recipient count is not a us-data target.", + "purpose": "n/a \u2014 TANF dollars are the target, not the recipient count.", + "verdict_basis": "deferred: no TANF-receipt indicator wired; the dollar target is compiled via cash_assistance." + } }, "hhs_acf_tanf.cash_assistance": { "status": "compiled" @@ -302,14 +491,24 @@ "hud.housing_assistance": { "status": "reviewed_exclusion", "classification": "source_absent", - "reason": "HUD housing-assistance aggregates targeted by the retired us-data pipeline (db/etl_housing_assistance.py). The pinned feed carries no HUD source fact; source-absent pending a Ledger HUD ingest.", - "evidence": "retired us-data pipeline (archived) db/etl_housing_assistance.py; feed carries no 'hud' source" + "reason": "HUD housing-assistance aggregates the retired pipeline calibrated (db/etl_housing_assistance.py). The pinned feed carries no HUD fact.", + "evidence": "retired us-data pipeline (archived) db/etl_housing_assistance.py; feed carries no 'hud' source", + "fence": { + "origin": "us-data db/etl_housing_assistance.py", + "purpose": "anchor modeled housing assistance to HUD admin totals.", + "verdict_basis": "source-absent: no HUD ledger fact in the feed; deferred pending a Ledger HUD ingest." + } }, "irs_soi.congressional_district_2022": { "status": "reviewed_exclusion", "classification": "off_by_default", - "reason": "IRS SOI congressional-district table. CD-level targets are opt-in (include_congressional_district_targets=False for the national release); the national and state SOI surfaces are compiled instead. Enabling CD targets compiles these \u2014 they are excluded for the national build by design, not dropped.", - "evidence": "test_soi_congressional_district_targets_are_opt_in + the include_congressional_district_targets gate in fiscal_targets._soi_reference_from_fact" + "reason": "IRS SOI congressional-district table. CD-level targets are opt-in (include_congressional_district_targets=False for the national release); the national and state SOI surfaces are compiled instead.", + "evidence": "test_soi_congressional_district_targets_are_opt_in + the include_congressional_district_targets gate in fiscal_targets._soi_reference_from_fact", + "fence": { + "origin": "us-data CD targets serve the regional/local H5 outputs (build_outputs/target_universe.py), not the national build.", + "purpose": "distributional shape within a congressional district for local H5 outputs.", + "verdict_basis": "off_by_default: CD targets are opt-in for the national release; enabling include_congressional_district_targets compiles them. The national and state SOI surfaces are compiled." + } }, "irs_soi.filing_season_week47": { "status": "compiled" @@ -317,20 +516,35 @@ "irs_soi.form_w2_401k_elective_deferrals": { "status": "reviewed_exclusion", "classification": "input_side", - "reason": "IRS SOI / W-2 retirement-contribution aggregate. Retirement contributions are modeled as imputed INPUT columns (traditional/roth 401(k) and IRA contributions), governed by the input-coverage contract, not as reweighting targets. Calibrating them as fiscal targets would double-govern a quantity the input-coverage gate already owns.", - "evidence": "RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py" + "reason": "IRS SOI / W-2 retirement-contribution aggregate the retired pipeline calibrated (PR #496/#554) but which it and populace both moved to the input side: modeled as imputed pre-limit contribution INPUT columns, not reweighting targets.", + "evidence": "POST_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py", + "fence": { + "origin": "us-data PR #496 (IRA targets) and PR #554 (401k/SE pension targets), converged at PR #1125", + "purpose": "ensure income-contribution consistency (\"$0-wage PUF clones get $0 in 401(k) contributions\", PR #554); target the ALD deduction not raw contributions because \"the variable flows directly into the ALD with no deductibility logic in policyengine-us, so the target must match the deduction, not total contributions\" (loss.py)", + "verdict_basis": "architecture retires it: us-data PR #1125 and populace both govern retirement contributions as imputed pre-limit INPUT columns (*_contributions_desired, POST_REFERENCE_ECPS_REQUIRED_INPUTS in the input-coverage contract) with PolicyEngine-US applying statutory limits \u2014 calibrating them here would double-govern a quantity the input-coverage gate owns. The feed carries the amounts, so this is not a source gap." + } }, "irs_soi.form_w2_designated_roth_401k_contributions": { "status": "reviewed_exclusion", "classification": "input_side", - "reason": "IRS SOI / W-2 retirement-contribution aggregate. Retirement contributions are modeled as imputed INPUT columns (traditional/roth 401(k) and IRA contributions), governed by the input-coverage contract, not as reweighting targets. Calibrating them as fiscal targets would double-govern a quantity the input-coverage gate already owns.", - "evidence": "RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py" + "reason": "IRS SOI / W-2 retirement-contribution aggregate the retired pipeline calibrated (PR #496/#554) but which it and populace both moved to the input side: modeled as imputed pre-limit contribution INPUT columns, not reweighting targets.", + "evidence": "POST_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py", + "fence": { + "origin": "us-data PR #496 (IRA targets) and PR #554 (401k/SE pension targets), converged at PR #1125", + "purpose": "ensure income-contribution consistency (\"$0-wage PUF clones get $0 in 401(k) contributions\", PR #554); target the ALD deduction not raw contributions because \"the variable flows directly into the ALD with no deductibility logic in policyengine-us, so the target must match the deduction, not total contributions\" (loss.py)", + "verdict_basis": "architecture retires it: us-data PR #1125 and populace both govern retirement contributions as imputed pre-limit INPUT columns (*_contributions_desired, POST_REFERENCE_ECPS_REQUIRED_INPUTS in the input-coverage contract) with PolicyEngine-US applying statutory limits \u2014 calibrating them here would double-govern a quantity the input-coverage gate owns. The feed carries the amounts, so this is not a source gap." + } }, "irs_soi.form_w2_social_security_tips": { "status": "reviewed_exclusion", "classification": "not_modeled", - "reason": "IRS W-2 Social Security tip aggregates. Already declared in US_FISCAL_TARGET_SUPPORT_EXCLUSIONS: current US support does not materialize a positive tip_income source column, so the W-2 tip return counts need the SIPP/ORG tip source stage wired before calibration.", - "evidence": "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS entry irs_soi.ty2023.form_w2_social_security_tips.box_7_social_security_tips.return_count" + "reason": "IRS W-2 Social Security tips \u2014 a real us-data target (PR #220) the model cannot yet satisfy: PolicyEngine-US produces a structural zero for tip_income in the populace base microdata (no tip source column), so the target is unsatisfiable until the tip-imputation source stage is ported.", + "evidence": "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS entry irs_soi.ty2023.form_w2_social_security_tips.box_7_social_security_tips.return_count", + "fence": { + "origin": "us-data PR #220 (Impute tips, fixes #215)", + "purpose": "\"Proposals such as 'No Tax on Tips' require a clean tip_income field distinct from regular wages ... Tipped workers skew lower-income; omitting tips biases poverty, EITC/CTC, and payroll-tax results\" (issue #215)", + "verdict_basis": "deferred (not source-absent \u2014 the feed carries a tip amount): PolicyEngine-US yields a structural zero for tip_income in the populace base microdata (US_FISCAL_TARGET_SUPPORT_EXCLUSIONS), so the target is unsatisfiable until the SIPP/ORG tip-imputation source stage (us-data #220) is ported. Wiring it now would ship a 0-vs-target gap, not a fit." + } }, "irs_soi.historic_table_2": { "status": "compiled" @@ -338,8 +552,13 @@ "irs_soi.roth_ira_contributions": { "status": "reviewed_exclusion", "classification": "input_side", - "reason": "IRS SOI / W-2 retirement-contribution aggregate. Retirement contributions are modeled as imputed INPUT columns (traditional/roth 401(k) and IRA contributions), governed by the input-coverage contract, not as reweighting targets. Calibrating them as fiscal targets would double-govern a quantity the input-coverage gate already owns.", - "evidence": "RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py" + "reason": "IRS SOI / W-2 retirement-contribution aggregate the retired pipeline calibrated (PR #496/#554) but which it and populace both moved to the input side: modeled as imputed pre-limit contribution INPUT columns, not reweighting targets.", + "evidence": "POST_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py", + "fence": { + "origin": "us-data PR #496 (IRA targets) and PR #554 (401k/SE pension targets), converged at PR #1125", + "purpose": "ensure income-contribution consistency (\"$0-wage PUF clones get $0 in 401(k) contributions\", PR #554); target the ALD deduction not raw contributions because \"the variable flows directly into the ALD with no deductibility logic in policyengine-us, so the target must match the deduction, not total contributions\" (loss.py)", + "verdict_basis": "architecture retires it: us-data PR #1125 and populace both govern retirement contributions as imputed pre-limit INPUT columns (*_contributions_desired, POST_REFERENCE_ECPS_REQUIRED_INPUTS in the input-coverage contract) with PolicyEngine-US applying statutory limits \u2014 calibrating them here would double-govern a quantity the input-coverage gate owns. The feed carries the amounts, so this is not a source gap." + } }, "irs_soi.state_2022": { "status": "compiled" @@ -365,8 +584,13 @@ "irs_soi.traditional_ira_contributions": { "status": "reviewed_exclusion", "classification": "input_side", - "reason": "IRS SOI / W-2 retirement-contribution aggregate. Retirement contributions are modeled as imputed INPUT columns (traditional/roth 401(k) and IRA contributions), governed by the input-coverage contract, not as reweighting targets. Calibrating them as fiscal targets would double-govern a quantity the input-coverage gate already owns.", - "evidence": "RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py" + "reason": "IRS SOI / W-2 retirement-contribution aggregate the retired pipeline calibrated (PR #496/#554) but which it and populace both moved to the input side: modeled as imputed pre-limit contribution INPUT columns, not reweighting targets.", + "evidence": "POST_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired input columns in release_input_coverage.py", + "fence": { + "origin": "us-data PR #496 (IRA targets) and PR #554 (401k/SE pension targets), converged at PR #1125", + "purpose": "ensure income-contribution consistency (\"$0-wage PUF clones get $0 in 401(k) contributions\", PR #554); target the ALD deduction not raw contributions because \"the variable flows directly into the ALD with no deductibility logic in policyengine-us, so the target must match the deduction, not total contributions\" (loss.py)", + "verdict_basis": "architecture retires it: us-data PR #1125 and populace both govern retirement contributions as imputed pre-limit INPUT columns (*_contributions_desired, POST_REFERENCE_ECPS_REQUIRED_INPUTS in the input-coverage contract) with PolicyEngine-US applying statutory limits \u2014 calibrating them here would double-govern a quantity the input-coverage gate owns. The feed carries the amounts, so this is not a source gap." + } }, "jct.tax_expenditures": { "status": "compiled" @@ -374,8 +598,13 @@ "kff.marketplace_effectuated_enrollment": { "status": "reviewed_exclusion", "classification": "superseded", - "reason": "Kaiser Family Foundation state marketplace effectuated-enrollment compilation. ACA marketplace enrollment is calibrated from the primary CMS administrative source (cms_aca.oep2024, compiled as cms_aca:102); KFF is a secondary aggregator of the same underlying CMS data, excluded to avoid a duplicate target.", - "evidence": "compiled sibling cms_aca.oep2024; sample fact kff.marketplace_effectuated_enrollment.2024.state.us.total_effectuated_marketplace_enrollment" + "reason": "Kaiser Family Foundation state marketplace enrollment \u2014 a secondary aggregator of the CMS data us-data calibrated. ACA enrollment is fit from the primary CMS source, compiled.", + "evidence": "compiled sibling cms_aca.oep2024; sample fact kff.marketplace_effectuated_enrollment.2024.state.us.total_effectuated_marketplace_enrollment", + "fence": { + "origin": "not a us-data calibration target: us-data calibrates ACA enrollment from CMS (nation/gov/aca_enrollment), not KFF.", + "purpose": "n/a \u2014 a secondary compilation of the same CMS data.", + "verdict_basis": "superseded by compiled cms_aca.oep2024 (the primary CMS administrative source); avoids a duplicate target." + } }, "ssa_supplement.oasdi_ssi_payments": { "status": "compiled" @@ -392,8 +621,13 @@ "usda_snap.national_average_monthly_persons": { "status": "reviewed_exclusion", "classification": "not_modeled", - "reason": "USDA FNS SNAP average-monthly PERSONS. The SNAP assistance unit is often a subset of the SPM unit (FY2024 FNS persons-per-household 1.88 vs 2.82 simulated members per taker unit), so a person indicator overcounts FNS participants by ~50%. The household caseload (usda_snap.national/state_average_monthly_households, compiled) is the calibrated count; persons stay unmapped until sub-unit participation is modeled.", - "evidence": "INDICATOR_LEDGER_TARGETS average_monthly_households comment + test_snap_person_caseload_fact_is_not_compiled" + "reason": "USDA FNS SNAP average-monthly PERSONS. A person indicator overcounts FNS participants ~50% (FY2024 persons-per-household 1.88 vs 2.82 simulated members); the household caseload is the calibrated count.", + "evidence": "INDICATOR_LEDGER_TARGETS average_monthly_households comment + test_snap_person_caseload_fact_is_not_compiled", + "fence": { + "origin": "not a us-data calibration target as a persons row: us-data calibrates SNAP dollars (CBO_PROGRAMS 'snap'), and the household caseload (usda_snap.*_average_monthly_households) is compiled.", + "purpose": "n/a \u2014 the household caseload is the model counterpart, not persons.", + "verdict_basis": "not_modeled: PolicyEngine-US does not model sub-unit SNAP participation, so a person indicator overcounts ~50% vs the compiled household caseload." + } }, "usda_snap.national_benefits": { "status": "compiled" @@ -404,8 +638,13 @@ "usda_snap.state_average_monthly_persons": { "status": "reviewed_exclusion", "classification": "not_modeled", - "reason": "USDA FNS SNAP average-monthly PERSONS. The SNAP assistance unit is often a subset of the SPM unit (FY2024 FNS persons-per-household 1.88 vs 2.82 simulated members per taker unit), so a person indicator overcounts FNS participants by ~50%. The household caseload (usda_snap.national/state_average_monthly_households, compiled) is the calibrated count; persons stay unmapped until sub-unit participation is modeled.", - "evidence": "INDICATOR_LEDGER_TARGETS average_monthly_households comment + test_snap_person_caseload_fact_is_not_compiled" + "reason": "USDA FNS SNAP average-monthly PERSONS. A person indicator overcounts FNS participants ~50% (FY2024 persons-per-household 1.88 vs 2.82 simulated members); the household caseload is the calibrated count.", + "evidence": "INDICATOR_LEDGER_TARGETS average_monthly_households comment + test_snap_person_caseload_fact_is_not_compiled", + "fence": { + "origin": "not a us-data calibration target as a persons row: us-data calibrates SNAP dollars (CBO_PROGRAMS 'snap'), and the household caseload (usda_snap.*_average_monthly_households) is compiled.", + "purpose": "n/a \u2014 the household caseload is the model counterpart, not persons.", + "verdict_basis": "not_modeled: PolicyEngine-US does not model sub-unit SNAP participation, so a person indicator overcounts ~50% vs the compiled household caseload." + } }, "usda_snap.state_benefits": { "status": "compiled" @@ -413,8 +652,13 @@ "wic.national_summary": { "status": "reviewed_exclusion", "classification": "source_absent", - "reason": "USDA WIC national annual summary targeted by the retired us-data pipeline (WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE in etl_national_targets). The pinned feed carries no WIC source fact; source-absent. WIC receipt is modeled via the would_claim_wic take-up input, not a target.", - "evidence": "retired us-data pipeline (archived) db/etl_national_targets.py WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE; feed carries no 'wic' source" + "reason": "USDA WIC national annual summary the retired pipeline calibrated (WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE). The pinned feed carries no WIC fact; WIC receipt is modeled via the would_claim_wic input.", + "evidence": "retired us-data pipeline (archived) db/etl_national_targets.py WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE; feed carries no 'wic' source", + "fence": { + "origin": "us-data db/etl_national_targets.py WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE", + "purpose": "anchor WIC participation to the USDA national summary.", + "verdict_basis": "source-absent: no WIC ledger fact in the feed. WIC receipt is modeled via the would_claim_wic take-up input, not a target here." + } } } } diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index 19a5b52a..90b0b99c 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -553,10 +553,11 @@ us_release_reform_coverage_probes, ) from populace.build.us_runtime.release_target_parity import ( - SSI_RECIPIENT_RED_LINE_FAMILIES, + RED_LINE_COMPILED_FAMILIES, US_TARGET_PARITY_FEED_FAMILIES_RESOURCE, US_TARGET_PARITY_MANIFEST_RESOURCE, TargetFamily, + TargetFence, TargetParityManifest, assert_target_parity_manifest_current, load_target_parity_feed_families, @@ -1617,10 +1618,11 @@ "us_release_input_coverage_required_columns", "us_release_input_coverage_reviewed_exclusions", "us_release_reform_coverage_probes", - "SSI_RECIPIENT_RED_LINE_FAMILIES", + "RED_LINE_COMPILED_FAMILIES", "US_TARGET_PARITY_FEED_FAMILIES_RESOURCE", "US_TARGET_PARITY_MANIFEST_RESOURCE", "TargetFamily", + "TargetFence", "TargetParityManifest", "assert_target_parity_manifest_current", "load_target_parity_feed_families", diff --git a/packages/populace-build/src/populace/build/us_runtime/fiscal_targets.py b/packages/populace-build/src/populace/build/us_runtime/fiscal_targets.py index 0622d050..0090b4d2 100644 --- a/packages/populace-build/src/populace/build/us_runtime/fiscal_targets.py +++ b/packages/populace-build/src/populace/build/us_runtime/fiscal_targets.py @@ -427,6 +427,17 @@ "cms_medicare", {"target_role": "medicare_part_b_premium_total"}, ), + # Household net worth: the retired us-data/eCPS pipeline calibrated a + # national wealth control (loss.py "net_worth", PR #282). Per-household + # net_worth is SCF-imputed and fixed, so the weighted sum is linear in the + # reweighting weights — a valid direct-sum target that pins the post-reweight + # aggregate the input-stage SCF imputation does not. Federal Reserve Z.1 + # series BOGZ1FL192090005Q (households & nonprofits net worth). + ("federal_reserve", "amount_outstanding", "fl152090005"): ( + "net_worth", + "federal_reserve", + {"target_role": "household_net_worth"}, + ), } @@ -506,6 +517,17 @@ "fact_aggregation": "time_mean", }, ), + # LIHEAP recipient households: the retired us-data/eCPS pipeline calibrated + # the ACF national recipient-household count (loss.py + # _add_liheap_targets_from_db, PR #688) as an indicator sum of the model's + # energy-subsidy receipt over SPM units — the exact SNAP-caseload pattern. + # Only the recipient COUNT is targeted; LIHEAP dollars stay deferred (the + # ACF component split is not a clean model concept, us-data PR #688). + ("hhs_acf_liheap", "households_served"): ( + "spm_unit_energy_subsidy", + "hhs_acf_liheap", + {"target_role": "liheap_households"}, + ), } @@ -2045,6 +2067,8 @@ def _reference_from_ledger_fact( ) if source_name == "ssa": return _ssa_ssi_reference_from_fact(fact, target_period=target_period) + if source_name == "bea": + return _bea_reference_from_fact(fact, target_period=target_period) if source_name == "jct": return None return _direct_reference_from_fact(fact, target_period=target_period) @@ -2446,6 +2470,98 @@ def _ssa_ssi_reference_from_fact( ) +# BEA all-population (nonfiler-inclusive) income targets the retired us-data/ +# eCPS pipeline calibrated to (us-data utils/loss.py +# BEA_NIPA_DIRECT_SUM_TARGETS, PR #994). Only the national wage and proprietors' +# income series are direct-sum targets here: us-data explicitly declined the +# NIPA interest/dividend totals because they include imputed interest, +# pension-plan dividends, and trust flows that are not a close microdata concept +# (PR #1059), and its state wage target (PR #1034) requires a place-of-work -> +# residence definitional adjustment the feed's raw SAINC4 line-50 facts cannot +# reproduce, so state wages are a documented reviewed exclusion. Every other BEA +# series in the feed is a ledger reference fact, not a calibration target. +_BEA_NATIONAL_WAGES_RECORD_SET = "bea_nipa.total_wages_salaries" +_BEA_NATIONAL_PROPRIETORS_RECORD_SET = "bea_nipa.proprietors_income" +#: The engine-computed counterparts us-data mapped these NIPA totals to. +_BEA_WAGES_BASE_VARIABLE = "employment_income_before_lsr" +_BEA_PROPRIETORS_BASE_VARIABLES = ( + "self_employment_income_before_lsr", + "sstb_self_employment_income_before_lsr", + "farm_operations_income", + "partnership_s_corp_income", +) +BEA_NIPA_WAGES_TARGET_ROLE = "nipa_wages_and_salaries" +BEA_NIPA_PROPRIETORS_TARGET_ROLE = "nipa_proprietors_income" + + +def _bea_reference_from_fact( + fact: object, + *, + target_period: int | str, +) -> LedgerTargetReference | None: + """Compile a BEA (``source_name == "bea"``) fact into a US target. + + Follows the retired us-data/eCPS mapping exactly (us-data + utils/loss.py ``BEA_NIPA_DIRECT_SUM_TARGETS``, PR #994): + + - ``bea_nipa.total_wages_salaries`` (BEA NIPA Table 2.1 gross wages and + salaries for all workers, **including nonfilers**; FRED A034RC) → + economy-wide engine ``employment_income_before_lsr``, national. This is + the ~$12.4T wage universe that filed tax returns and CPS-reported wages + (~$4T in the base microdata) systematically undercount — the nonfiler / + unreported-wage coverage gap the target was added to close (PR #994). + - ``bea_nipa.proprietors_income`` (BEA NIPA Table 2.1 proprietors' income + with IVA and CCAdj, all persons; FRED A041RC) → the same additive + expression us-data used: Schedule C non-SSTB and SSTB self-employment + income before labor-supply responses, farm operations income, and active + partnership/S-corp income, national. + + Only these two national record sets compile; every other BEA series returns + ``None`` (a reviewed exclusion in the target-parity manifest, whose fence + records the NIPA interest/dividend decline and the state-wage + residence-adjustment gap). + """ + record_set_id = _normalized_record_set_id(_str_at(fact, "layout", "record_set_id")) + if _geography_level(fact) != "country": + return None + + if record_set_id == _BEA_NATIONAL_WAGES_RECORD_SET: + base_variable: str | tuple[str, ...] = _BEA_WAGES_BASE_VARIABLE + target_role = BEA_NIPA_WAGES_TARGET_ROLE + elif record_set_id == _BEA_NATIONAL_PROPRIETORS_RECORD_SET: + base_variable = _BEA_PROPRIETORS_BASE_VARIABLES + target_role = BEA_NIPA_PROPRIETORS_TARGET_ROLE + else: + return None + + source_record_id = _source_record_id(fact) + if not source_record_id: + return None + + metadata = { + "materializer": "policyengine_variable", + "measure_mode": "sum", + "target_role": target_role, + "source_measure_id": _measure_id(fact), + "source_period": str(_period_value(fact)), + "target_period": str(target_period), + } + if isinstance(base_variable, tuple): + metadata["base_variables"] = ",".join(base_variable) + else: + metadata["base_variable"] = base_variable + return LedgerTargetReference( + name=source_record_id, + ledger_source_record_id=source_record_id, + entity="household", + measure=source_record_id, + period=target_period, + family="bea", + signed=_numeric_value(fact) < 0, + metadata=metadata, + ) + + def _direct_reference_from_fact( fact: object, *, diff --git a/packages/populace-build/src/populace/build/us_runtime/release_target_parity.py b/packages/populace-build/src/populace/build/us_runtime/release_target_parity.py index aab9583a..dc3170c1 100644 --- a/packages/populace-build/src/populace/build/us_runtime/release_target_parity.py +++ b/packages/populace-build/src/populace/build/us_runtime/release_target_parity.py @@ -61,8 +61,9 @@ "COMPILED_STATUS", "REVIEWED_EXCLUSION_STATUS", "SOURCE_ABSENT_CLASSIFICATION", - "SSI_RECIPIENT_RED_LINE_FAMILIES", + "RED_LINE_COMPILED_FAMILIES", "TargetFamily", + "TargetFence", "TargetParityManifest", "assert_target_parity_manifest_current", "load_target_parity_feed_families", @@ -88,13 +89,23 @@ #: declared families exempt from the feed-surface reconciliation. SOURCE_ABSENT_CLASSIFICATION = "source_absent" -#: The core SSA SSI recipient family. Per this contract it must ship -#: ``compiled`` with no reviewed exclusion — the whole point of the register is -#: that the family the retired pipeline calibrated (``nation/ssa/ssi_recipients``) -#: can never again be silently dropped. The anti-rot check refuses to let it be -#: downgraded, mirroring the #368 SSI countable-resource asset red line on the -#: input-coverage side. -SSI_RECIPIENT_RED_LINE_FAMILIES = ("ssa_supplement.ssi_recipients",) +#: Families that must ship ``compiled`` with no reviewed exclusion — the whole +#: point of the register is that the administrative targets the retired pipeline +#: calibrated can never again be silently dropped. The anti-rot check refuses to +#: let any of them be downgraded, mirroring the #368 SSI countable-resource asset +#: red line on the input-coverage side. +#: +#: - ``ssa_supplement.ssi_recipients``: the SSA SSI recipient count +#: (``nation/ssa/ssi_recipients``). +#: - ``bea_nipa.total_wages_salaries``: the BEA NIPA all-population wage total +#: (``nation/bea/nipa_wages_and_salaries``, PR #994) — economy-wide wages +#: including nonfilers, the ~$12.4T universe that tax-return / CPS-reported +#: wages undercount by two-thirds. Silent loss of it is the exact failure the +#: Chesterton's-fence audit surfaced. +RED_LINE_COMPILED_FAMILIES = ( + "ssa_supplement.ssi_recipients", + "bea_nipa.total_wages_salaries", +) def _is_target_period_token(value: str) -> bool: @@ -144,6 +155,42 @@ def us_target_family_id(source_record_id: str) -> str: return namespace +@dataclass(frozen=True) +class TargetFence: + """The Chesterton's-fence record behind one reviewed exclusion. + + A category label (``macro_control_total``, ``not_modeled``, ``superseded``) + is not a sufficient reason to drop a target the retired pipeline calibrated + to. Before an exclusion may stand it must recover *why the fence was built*: + the target's origin, the failure mode it guarded, and the purpose-informed + basis for not rebuilding it here. An unexplained fence gets rebuilt (wired), + not removed. + + Attributes: + origin: Where the fence came from — the introducing (and, where + relevant, removing) ``us-data`` PR/commit, or the + explicit finding ``"not a us-data calibration target"`` when the feed + fact was never a target (a ledger reference fact). + purpose: The failure mode the target guarded, quoting the PR/issue + rationale verbatim where one exists, or stating ``"no discoverable + rationale in PR #N"`` (then the exclusion must justify itself on + mechanics), or ``"n/a"`` for a fact that was never a target. + verdict_basis: The purpose-informed reason the exclusion stands — a + named compiled family that subsumes it, an architecture change that + retires the need (with a code/PR cite), or a deferral with the + specific blocker. Never a bare category label. + """ + + origin: str + purpose: str + verdict_basis: str + + def __post_init__(self) -> None: + for field_name in ("origin", "purpose", "verdict_basis"): + if not getattr(self, field_name): + raise ValueError(f"TargetFence.{field_name} is required.") + + @dataclass(frozen=True) class TargetFamily: """One declared target-parity family. @@ -155,13 +202,16 @@ class TargetFamily: classification: For a reviewed exclusion, the exclusion kind (``survey_derived``, ``macro_control_total``, ``non_linear``, ``off_by_default``, ``superseded``, ``source_absent``, - ``deferred``, ``input_side``, ``not_modeled``). Empty for a - compiled family. + ``deferred``, ``input_side``, ``not_modeled``, ``not_a_target``). + A classification is now a label ON TOP of the fence narrative, not a + reason by itself. Empty for a compiled family. reason: Why the gap is accepted — required for a reviewed exclusion. evidence: The concrete fact/mechanism the reason names (a sample ``source_record_id``, a code constant, a compiled sibling family). - Required for a reviewed exclusion — an undocumented exclusion is a - silent omission with extra steps. + Required for a reviewed exclusion. + fence: The Chesterton's-fence record (origin, purpose, verdict_basis). + REQUIRED for every reviewed exclusion — a category label alone is no + longer a sufficient reason to drop a us-data-era target. issue: Optional tracking issue owning the gap's closure. note: Optional free-text annotation. """ @@ -171,6 +221,7 @@ class TargetFamily: classification: str = "" reason: str = "" evidence: str = "" + fence: TargetFence | None = None issue: str = "" note: str = "" @@ -197,6 +248,12 @@ def __post_init__(self) -> None: f"{self.name}: a reviewed exclusion needs evidence naming the " "concrete fact or mechanism behind the reason." ) + if self.fence is None: + raise ValueError( + f"{self.name}: a reviewed exclusion needs a fence " + "{origin, purpose, verdict_basis} — a category label is not a " + "sufficient reason to drop a us-data-era calibration target." + ) @property def is_source_absent(self) -> bool: @@ -317,6 +374,18 @@ def load_target_parity_manifest( for name, entry in sorted(raw_families.items()): if not isinstance(entry, Mapping): raise ValueError(f"{resource}: family {name!r} must be a JSON object.") + raw_fence = entry.get("fence") + fence: TargetFence | None = None + if raw_fence is not None: + if not isinstance(raw_fence, Mapping): + raise ValueError( + f"{resource}: family {name!r} 'fence' must be a JSON object." + ) + fence = TargetFence( + origin=str(raw_fence.get("origin", "")), + purpose=str(raw_fence.get("purpose", "")), + verdict_basis=str(raw_fence.get("verdict_basis", "")), + ) families.append( TargetFamily( name=str(name), @@ -324,6 +393,7 @@ def load_target_parity_manifest( classification=str(entry.get("classification", "")), reason=str(entry.get("reason", "")), evidence=str(entry.get("evidence", "")), + fence=fence, issue=str(entry.get("issue", "")), note=str(entry.get("note", "")), ) @@ -449,7 +519,7 @@ def assert_target_parity_manifest_current( is drift and must be reconciled by regenerating the manifest. - The manifest's ``reference.feed_sha256`` must match the inventory's, so the two artifacts describe the same pinned feed. - - The core SSA SSI recipient family must stay ``compiled`` with no reviewed + - The red-line families must stay ``compiled`` with no reviewed exclusion — the red line this contract exists for cannot be quietly undone. - When a compiled registry is supplied, every ``compiled`` family must be present in it, every ``reviewed_exclusion`` family must be absent from it, @@ -492,16 +562,16 @@ def assert_target_parity_manifest_current( f"{inventory_sha!r}; regenerate both from the same pinned feed." ) - for family in SSI_RECIPIENT_RED_LINE_FAMILIES: + for family in RED_LINE_COMPILED_FAMILIES: entry = manifest.by_name.get(family) if entry is None: failures.append( - f"{family}: core SSA SSI recipient family must be declared and " - "compiled (the target-parity red line)." + f"{family}: red-line administrative target family must be declared " + "and compiled (the target-parity red line)." ) elif entry.status != COMPILED_STATUS: failures.append( - f"{family}: core SSA SSI recipient family must stay status=" + f"{family}: red-line administrative target family must stay status=" f"{COMPILED_STATUS!r}, not {entry.status!r} — it can never be " "quietly downgraded to a reviewed exclusion." ) diff --git a/packages/populace-build/src/populace/build/us_runtime/target_aging.py b/packages/populace-build/src/populace/build/us_runtime/target_aging.py index 217c77c1..fe84a0db 100644 --- a/packages/populace-build/src/populace/build/us_runtime/target_aging.py +++ b/packages/populace-build/src/populace/build/us_runtime/target_aging.py @@ -124,6 +124,13 @@ "cbo_qualified_dividend_income": "qualified_dividend_income", "cbo_net_capital_gain": "net_capital_gain", "cbo_net_business_income": "net_business_income", + # BEA NIPA all-population income targets (nonfiler-inclusive) age by their + # own CBO income-by-source series so a cy2023 source level is projected to + # the build year on the matching concept, not the AGI default. The cy2024 + # NIPA wage level needs no aging (source == build period). + "nipa_wages_and_salaries": "wages_and_salaries", + "bea_state_wages": "wages_and_salaries", + "nipa_proprietors_income": "net_business_income", } diff --git a/packages/populace-build/tests/test_release_target_parity.py b/packages/populace-build/tests/test_release_target_parity.py index 10759c51..3c47ab48 100644 --- a/packages/populace-build/tests/test_release_target_parity.py +++ b/packages/populace-build/tests/test_release_target_parity.py @@ -29,9 +29,10 @@ from populace.build.us_runtime.release_target_parity import ( COMPILED_STATUS, + RED_LINE_COMPILED_FAMILIES, REVIEWED_EXCLUSION_STATUS, - SSI_RECIPIENT_RED_LINE_FAMILIES, TargetFamily, + TargetFence, TargetParityManifest, assert_target_parity_manifest_current, load_target_parity_feed_families, @@ -41,6 +42,12 @@ us_target_family_id, ) +_FENCE = TargetFence( + origin="not a us-data calibration target (test)", + purpose="n/a — test fixture", + verdict_basis="no fence to rebuild (test)", +) + _REPO_ROOT = Path(__file__).resolve().parents[3] _MANIFEST_GENERATOR = _REPO_ROOT / "tools" / "build_us_target_parity_manifest.py" _US_PACKAGE_DIR = ( @@ -71,6 +78,7 @@ def _manifest(families: tuple[TargetFamily, ...]) -> TargetParityManifest: _CONTRACT = _manifest( ( TargetFamily("ssa_supplement.ssi_recipients", COMPILED_STATUS), + TargetFamily("bea_nipa.total_wages_salaries", COMPILED_STATUS), TargetFamily("usda_snap.state_benefits", COMPILED_STATUS), TargetFamily( "bea_nipa.personal_income", @@ -78,10 +86,29 @@ def _manifest(families: tuple[TargetFamily, ...]) -> TargetParityManifest: classification="macro_control_total", reason="NIPA macro aggregate; income fit from SOI + CBO.", evidence="compiled sibling irs_soi.historic_table_2", + fence=_FENCE, ), ) ) +#: The feed inventory consistent with ``_CONTRACT`` (every compiled + reviewed +#: family present, both red-line families declared). +_CONTRACT_FEED = { + "feed_sha256": "abc", + "families": { + "ssa_supplement.ssi_recipients": 52, + "bea_nipa.total_wages_salaries": 1, + "usda_snap.state_benefits": 51, + "bea_nipa.personal_income": 1, + }, +} + +_CONTRACT_COMPILED = ( + "ssa_supplement.ssi_recipients", + "bea_nipa.total_wages_salaries", + "usda_snap.state_benefits", +) + class TestFamilyId: def test_ssa_recipients(self) -> None: @@ -116,13 +143,11 @@ def test_empty(self) -> None: class TestGate: def test_every_compiled_family_present_passes(self) -> None: - registry = _registry( - ["ssa_supplement.ssi_recipients", "usda_snap.state_benefits"] - ) + registry = _registry(_CONTRACT_COMPILED) result = us_release_target_parity_gate(registry, manifest=_CONTRACT) assert result.passed assert result.name == "us_release_target_parity" - assert result.details["compiled_families"] == 2 + assert result.details["compiled_families"] == 3 def test_missing_compiled_family_fails_named(self) -> None: registry = _registry(["ssa_supplement.ssi_recipients"]) @@ -132,13 +157,7 @@ def test_missing_compiled_family_fails_named(self) -> None: def test_stale_reviewed_exclusion_fails(self) -> None: # A reviewed-exclusion family the registry now compiles must be promoted. - registry = _registry( - [ - "ssa_supplement.ssi_recipients", - "usda_snap.state_benefits", - "bea_nipa.personal_income", - ] - ) + registry = _registry([*_CONTRACT_COMPILED, "bea_nipa.personal_income"]) result = us_release_target_parity_gate(registry, manifest=_CONTRACT) assert not result.passed assert any("bea_nipa.personal_income" in failure for failure in result.failures) @@ -162,6 +181,20 @@ def test_reviewed_exclusion_requires_evidence(self) -> None: "x.y", REVIEWED_EXCLUSION_STATUS, classification="c", reason="r" ) + def test_reviewed_exclusion_requires_fence(self) -> None: + with pytest.raises(ValueError, match="needs a fence"): + TargetFamily( + "x.y", + REVIEWED_EXCLUSION_STATUS, + classification="c", + reason="r", + evidence="e", + ) + + def test_fence_requires_all_three_fields(self) -> None: + with pytest.raises(ValueError, match="TargetFence.verdict_basis is required"): + TargetFence(origin="o", purpose="p", verdict_basis="") + def test_unknown_status_rejected(self) -> None: with pytest.raises(ValueError, match="status must be one of"): TargetFamily("x.y", "maybe") @@ -169,7 +202,10 @@ def test_unknown_status_rejected(self) -> None: class TestAntiRot: def test_undeclared_feed_family_fails(self) -> None: - feed = {"feed_sha256": "abc", "families": {"new_source.new_concept": 3}} + feed = { + "feed_sha256": "abc", + "families": {**_CONTRACT_FEED["families"], "new_source.new_concept": 3}, + } with pytest.raises(ValueError, match="not declared in the manifest"): assert_target_parity_manifest_current( manifest=_CONTRACT, feed_families=feed @@ -178,33 +214,22 @@ def test_undeclared_feed_family_fails(self) -> None: def test_feed_surface_family_no_longer_in_feed_fails(self) -> None: # usda_snap.state_benefits is a declared compiled (feed-surface) family; # a feed inventory omitting it is drift. - feed = { - "feed_sha256": "abc", - "families": { - "ssa_supplement.ssi_recipients": 52, - "bea_nipa.personal_income": 1, - }, - } + families = dict(_CONTRACT_FEED["families"]) + del families["usda_snap.state_benefits"] + feed = {"feed_sha256": "abc", "families": families} with pytest.raises(ValueError, match="no longer carries"): assert_target_parity_manifest_current( manifest=_CONTRACT, feed_families=feed ) def test_feed_sha_mismatch_fails(self) -> None: - feed = { - "feed_sha256": "different", - "families": { - "ssa_supplement.ssi_recipients": 52, - "usda_snap.state_benefits": 51, - "bea_nipa.personal_income": 1, - }, - } + feed = {**_CONTRACT_FEED, "feed_sha256": "different"} with pytest.raises(ValueError, match="does not match the feed-family"): assert_target_parity_manifest_current( manifest=_CONTRACT, feed_families=feed ) - def test_ssi_recipients_downgrade_is_rejected(self) -> None: + def test_red_line_downgrade_is_rejected(self) -> None: downgraded_families = tuple( replace( family, @@ -212,44 +237,23 @@ def test_ssi_recipients_downgrade_is_rejected(self) -> None: classification="deferred", reason="pretend we dropped it", evidence="none", + fence=_FENCE, ) - if family.name in SSI_RECIPIENT_RED_LINE_FAMILIES + if family.name in RED_LINE_COMPILED_FAMILIES else family for family in _CONTRACT.families ) downgraded = _manifest(downgraded_families) - feed = { - "feed_sha256": "abc", - "families": { - "ssa_supplement.ssi_recipients": 52, - "usda_snap.state_benefits": 51, - "bea_nipa.personal_income": 1, - }, - } with pytest.raises(ValueError, match="must stay status='compiled'"): assert_target_parity_manifest_current( - manifest=downgraded, feed_families=feed + manifest=downgraded, feed_families=_CONTRACT_FEED ) def test_registry_half_flags_undeclared_registry_family(self) -> None: - registry = _registry( - [ - "ssa_supplement.ssi_recipients", - "usda_snap.state_benefits", - "surprise.family", - ] - ) - feed = { - "feed_sha256": "abc", - "families": { - "ssa_supplement.ssi_recipients": 52, - "usda_snap.state_benefits": 51, - "bea_nipa.personal_income": 1, - }, - } + registry = _registry([*_CONTRACT_COMPILED, "surprise.family"]) with pytest.raises(ValueError, match="does not declare"): assert_target_parity_manifest_current( - manifest=_CONTRACT, feed_families=feed, registry=registry + manifest=_CONTRACT, feed_families=_CONTRACT_FEED, registry=registry ) @@ -260,19 +264,39 @@ def test_manifest_loads_nonempty_with_compiled_and_reviewed(self) -> None: assert manifest.reviewed_exclusions assert manifest.schema_version == 1 - def test_every_reviewed_exclusion_carries_reason_and_evidence(self) -> None: + def test_every_reviewed_exclusion_carries_reason_evidence_and_fence(self) -> None: manifest = load_target_parity_manifest() for family in manifest.families: if family.status == REVIEWED_EXCLUSION_STATUS: assert family.reason, family.name assert family.classification, family.name assert family.evidence, family.name + assert family.fence is not None, family.name + assert family.fence.origin, family.name + assert family.fence.purpose, family.name + assert family.fence.verdict_basis, family.name - def test_ssi_recipient_family_is_compiled(self) -> None: + def test_red_line_families_are_compiled(self) -> None: manifest = load_target_parity_manifest() - for family in SSI_RECIPIENT_RED_LINE_FAMILIES: + for family in RED_LINE_COMPILED_FAMILIES: assert manifest.by_name[family].status == COMPILED_STATUS + def test_wired_nipa_and_liheap_families_are_compiled(self) -> None: + manifest = load_target_parity_manifest() + for family in ( + "bea_nipa.total_wages_salaries", + "bea_nipa.proprietors_income", + "federal_reserve_z1.households_nonprofits_balance_sheet", + "hhs_acf_liheap.national_profile", + ): + assert manifest.by_name[family].status == COMPILED_STATUS + + def test_deferred_state_wages_carry_a_fence(self) -> None: + manifest = load_target_parity_manifest() + state_wages = manifest.by_name["bea_regional.state_wages_salaries"] + assert state_wages.status == REVIEWED_EXCLUSION_STATUS + assert "PR #1034" in state_wages.fence.origin + def test_ssi_state_payments_family_is_compiled(self) -> None: manifest = load_target_parity_manifest() assert manifest.by_name["ssa_supplement.ssi_payments"].status == COMPILED_STATUS diff --git a/packages/populace-build/tests/test_us_fiscal_targets.py b/packages/populace-build/tests/test_us_fiscal_targets.py index 66c2889c..406a1ee6 100644 --- a/packages/populace-build/tests/test_us_fiscal_targets.py +++ b/packages/populace-build/tests/test_us_fiscal_targets.py @@ -1308,6 +1308,171 @@ def test_ssa_oasdi_ssi_payment_aggregates_still_compile() -> None: assert ssi_payments.metadata["measure_mode"] == "sum" +def _bea_national_fact( + *, + record_set_concept: str, + measure_id: str, + groupby_value_id: str, + value: float, +) -> dict[str, object]: + record_set_id = f"bea_nipa.cy2024.{record_set_concept}" + source_record_id = f"{record_set_id}.{groupby_value_id}.{measure_id}" + return _dynamic_ledger_fact( + source_record_id=source_record_id, + source_name="bea", + measure_id=measure_id, + value=value, + groupby_value_id=groupby_value_id, + layout_record_set_id=record_set_id, + ) + + +def test_bea_nipa_wages_map_to_employment_income_before_lsr() -> None: + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + _bea_national_fact( + record_set_concept="total_wages_salaries", + measure_id="wages_salaries_amount", + groupby_value_id="a034rc", + value=12_387_929_000_000, + ), + ] + ) + + spec = {spec.name: spec for spec in registry.specs}[ + "bea_nipa.cy2024.total_wages_salaries.a034rc.wages_salaries_amount" + ] + assert spec.family == "bea" + assert spec.value == 12_387_929_000_000 + assert spec.metadata["base_variable"] == "employment_income_before_lsr" + assert spec.metadata["target_role"] == "nipa_wages_and_salaries" + assert spec.metadata["measure_mode"] == "sum" + + +def test_bea_nipa_proprietors_map_to_self_employment_expression() -> None: + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + _bea_national_fact( + record_set_concept="proprietors_income", + measure_id="amount", + groupby_value_id="a041rc", + value=2_023_080_000_000, + ), + ] + ) + + spec = {spec.name: spec for spec in registry.specs}[ + "bea_nipa.cy2024.proprietors_income.a041rc.amount" + ] + assert spec.family == "bea" + assert spec.metadata["base_variables"] == ( + "self_employment_income_before_lsr,sstb_self_employment_income_before_lsr," + "farm_operations_income,partnership_s_corp_income" + ) + assert spec.metadata["target_role"] == "nipa_proprietors_income" + + +def test_bea_other_nipa_series_are_not_compiled() -> None: + # Only wages and proprietors are direct-sum NIPA targets (loss.py PR #994); + # every other BEA series is a ledger reference fact, not a target. + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + _bea_national_fact( + record_set_concept="personal_income", + measure_id="amount", + groupby_value_id="a065rc", + value=20_000_000_000_000, + ), + ] + ) + + assert not [spec for spec in registry.specs if "personal_income" in spec.name] + + +def test_bea_state_wages_are_deferred_not_compiled() -> None: + # Raw place-of-work state wages need us-data's residence adjustment (PR #1034) + # the feed cannot reproduce; the state distribution is deferred. + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + _dynamic_ledger_fact( + source_record_id=("bea_regional.cy2024.state_wages_salaries.al.amount"), + source_name="bea", + measure_id="amount", + value=137_000_000_000, + geography_level="state", + geography_id="0400000US01", + groupby_value_id="al", + layout_record_set_id="bea_regional.cy2024.state_wages_salaries", + ), + ] + ) + + assert not [spec for spec in registry.specs if "state_wages_salaries" in spec.name] + + +def test_federal_reserve_net_worth_maps_to_net_worth_variable() -> None: + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + _dynamic_ledger_fact( + source_record_id=( + "federal_reserve_z1.cy2024.households_nonprofits_balance_sheet" + ".net_worth.fl152090005.amount_outstanding" + ), + source_name="federal_reserve", + measure_id="amount_outstanding", + value=160_000_000_000_000, + groupby_value_id="fl152090005", + layout_record_set_id=( + "federal_reserve_z1.cy2024.households_nonprofits_balance_sheet" + ".net_worth" + ), + ), + ] + ) + + spec = {spec.name: spec for spec in registry.specs}[ + "federal_reserve_z1.cy2024.households_nonprofits_balance_sheet" + ".net_worth.fl152090005.amount_outstanding" + ] + assert spec.family == "federal_reserve" + assert spec.value == 160_000_000_000_000 + assert spec.metadata["base_variable"] == "net_worth" + assert spec.metadata["target_role"] == "household_net_worth" + + +def test_liheap_households_map_to_energy_subsidy_indicator() -> None: + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + _dynamic_ledger_fact( + source_record_id=( + "hhs_acf_liheap.fy2024.national_profile.state_programs" + ".households_served" + ), + source_name="hhs_acf_liheap", + measure_id="households_served", + value=5_876_646, + groupby_value_id="state_programs", + layout_record_set_id="hhs_acf_liheap.fy2024.national_profile", + ), + ] + ) + + spec = {spec.name: spec for spec in registry.specs}[ + "hhs_acf_liheap.fy2024.national_profile.state_programs.households_served" + ] + assert spec.family == "hhs_acf_liheap" + assert spec.value == 5_876_646 + assert spec.metadata["base_variable"] == "spm_unit_energy_subsidy" + assert spec.metadata["target_role"] == "liheap_households" + assert spec.metadata["measure_mode"] == "indicator_sum" + + def test_dynamic_us_fiscal_targets_use_builder_target_period() -> None: source_record_id = "irs_soi.ty2023.table_3_3.us.all.income_tax_liability_amount" diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 59b3ee80..ce32b801 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -1,36 +1,33 @@ """Regenerate the US release target-parity manifest and feed-family inventory. The target-side analog of ``tools/build_us_release_input_coverage_manifest.py``. -Where that tool declares which reference-eCPS input columns a release must -persist, this one declares which administrative target *families* the compiled -populace registry must carry — the families the retired us-data/eCPS pipeline -calibrated to. - -Derivation (fully from the sha-pinned consumer feed + the deterministic registry -compile): - -- **Feed-family inventory** = every ``namespace.concept`` family id the pinned - ``consumer_facts_*.jsonl`` carries, with its fact count. Written to - ``target_parity_feed_families.json`` so the manifest consistency check runs in - CI without the 131 MB feed. +It declares which administrative target *families* the compiled populace registry +must carry — the families the retired us-data/eCPS pipeline calibrated to. + +Chesterton's fence is the governing rule. A category label (``macro_control_total``, +``not_modeled``, ``superseded``) is not a sufficient reason to drop a target the +retired pipeline calibrated to. Every reviewed exclusion must carry a ``fence`` +recovering *why the fence was built*: the target's origin (the introducing +us-data PR/commit, recovered by archaeology), the failure mode it +guarded (the PR/issue rationale, quoted where one exists), and the +purpose-informed verdict basis for not rebuilding it here. The absolute rule is +**us-data-targeted ⇒ compiled, unless source-absent**; ``macro_control_total`` is +only valid for families us-data did NOT target. Where archaeology is +inconclusive (no discoverable rationale), the fence is rebuilt — the target is +wired, not excluded. + +Derivation (from the sha-pinned feed + the deterministic registry compile): + +- **Feed-family inventory** = every ``namespace.concept`` family id the feed + carries, with fact counts (``target_parity_feed_families.json``). - **Compiled families** = the families the registry compiles today (``compile_us_fiscal_target_registry(age_targets=True)`` + the reviewed CMS - Medicaid enrollment substitution, exactly as the release builder compiles it). - Every compiled family gets ``status: compiled``. -- **Reviewed exclusions** = every feed family that does NOT compile, each - classified with a reason naming its evidence (a sample ``source_record_id``, a - code constant, or the compiled sibling family that supersedes it). A feed - family with neither a compile nor a declared exclusion HALTS generation — the - anti-rot guarantee that a new administrative family cannot be silently ignored. -- **Source-absent us-data families** = administrative targets the retired - us-data pipeline calibrated to for which the pinned feed carries no ledger - fact (BLS Consumer Expenditure, WIC, HUD housing assistance). Declared as - ``source_absent`` reviewed exclusions so the diff records them. + Medicaid enrollment substitution). Status ``compiled``. +- **Reviewed exclusions** = every non-compiling feed family, each with a fence. + A feed family with neither a compile nor a fenced exclusion HALTS generation. +- **Source-absent us-data families** = us-data targets with no feed fact. Run: uv run python tools/build_us_target_parity_manifest.py -It rewrites packages/populace-build/src/populace/build/us/target_parity_manifest.json -and target_parity_feed_families.json. A test asserts the committed files match -this regeneration, so the manifest cannot silently drift from the pinned feed. """ from __future__ import annotations @@ -59,7 +56,6 @@ MANIFEST_PATH = US_PACKAGE_DIR / "target_parity_manifest.json" FEED_FAMILIES_PATH = US_PACKAGE_DIR / "target_parity_feed_families.json" -#: The pinned consumer feed the certified registry compiles from. DEFAULT_FEED_PATH = ( Path.home() / "PolicyEngine" @@ -71,160 +67,344 @@ EXPECTED_FEED_SHA256_PREFIX = "94b7155f" TARGET_PERIOD = 2024 -# --------------------------------------------------------------------------- -# Reviewed exclusions — every feed family that does not compile, classified with -# an evidence-naming reason. Namespace rules cover the many one-per-series BEA -# aggregates; exact rules cover the rest. Reasons cite a compiled sibling -# family, a code constant, or the mechanism that makes the family non-linear / -# survey-derived / off-by-default. -# --------------------------------------------------------------------------- -# (classification, reason, evidence) keyed by source_record_id namespace. -_NAMESPACE_EXCLUSIONS: dict[str, tuple[str, str, str]] = { +def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: + return {"origin": origin, "purpose": purpose, "verdict_basis": verdict_basis} + + +# The fence for every BEA series the archaeology confirmed us-data never +# calibrated to (only nipa_wages_and_salaries + proprietors_income were +# direct-sum targets, PR #994; both are now compiled). The programs some NIPA +# series measure ARE calibrated by us-data, but from SSA/CMS/CBO administrative +# sources (compiled), never the BEA NIPA aggregate. +_BEA_NOT_A_TARGET_FENCE = _fence( + origin=( + "not a us-data calibration target: absent from us-data " + "utils/loss.py and db/etl_national_targets.py (archaeology). The only BEA " + "direct-sum targets were nipa_wages_and_salaries and proprietors_income " + "(PR #994), both now compiled." + ), + purpose="n/a — a ledger reference fact carried by the feed, never a target.", + verdict_basis=( + "no fence to rebuild. Programs some of these NIPA aggregates measure " + "(Social Security, Medicare, Medicaid, unemployment) are calibrated from " + "SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." + ), +) + +# (classification, reason, evidence, fence) keyed by source_record_id namespace. +_NAMESPACE_EXCLUSIONS: dict[str, tuple[str, str, str, dict[str, str]]] = { "bea_nipa": ( "macro_control_total", - "BEA NIPA national-accounts aggregate (calendar-year macro control " - "total). Not a household-linear administrative level PolicyEngine-US " - "calibrates against: household income components are fit from the IRS " - "SOI micro tables (irs_soi.historic_table_2, compiled) and CBO " - "revenue-by-source projections (cbo.revenue_projection, compiled). The " - "NIPA aggregate is retained in the ledger as a macro cross-check, not a " - "calibration target.", - "compiled siblings irs_soi.historic_table_2 and cbo.revenue_projection " - "supply the household-linear income surface; sample fact " - "bea_nipa.cy2023.personal_income.a065rc.amount", + "BEA NIPA national-accounts aggregate the retired pipeline did not " + "calibrate to (only wages and proprietors' income were NIPA direct-sum " + "targets, both compiled). A macro control total retained as a ledger " + "cross-check.", + "compiled BEA targets bea_nipa.total_wages_salaries and " + "bea_nipa.proprietors_income carry the household-linear NIPA income; " + "sample fact bea_nipa.cy2023.personal_income.a065rc.amount", + _BEA_NOT_A_TARGET_FENCE, ), "bea_regional": ( "macro_control_total", - "BEA regional (state) personal-income account aggregate. State income " - "is calibrated from the IRS SOI state tables (irs_soi.state_2022 and " - "the historic_table_2 state rows, compiled) and Census STC " - "(census_stc.individual_income_tax_collections, compiled); the BEA " - "regional total is a macro state control without a per-record model " - "counterpart.", + "BEA regional (state) account aggregate us-data did not calibrate to " + "(only state wages were a regional target, and it is deferred). State " + "income is fit from IRS SOI state tables and Census STC, compiled.", "compiled siblings irs_soi.state_2022 and " "census_stc.individual_income_tax_collections; sample fact " "bea_regional.cy2023.state_personal_income.us.amount", + _fence( + origin=( + "not a us-data calibration target: absent from loss.py and " + "etl_national_targets.py (archaeology). The only targeted " + "bea_regional series is state_wages_salaries (PR #1034), a " + "separate deferred exclusion." + ), + purpose="n/a — a ledger reference fact, never a target.", + verdict_basis=( + "no fence to rebuild; state income is calibrated from IRS SOI " + "state tables + Census STC (compiled)." + ), + ), ), } _RETIREMENT_CONTRIBUTION_EXCLUSION = ( "input_side", - "IRS SOI / W-2 retirement-contribution aggregate. Retirement contributions " - "are modeled as imputed INPUT columns (traditional/roth 401(k) and IRA " - "contributions), governed by the input-coverage contract, not as reweighting " - "targets. Calibrating them as fiscal targets would double-govern a quantity " - "the input-coverage gate already owns.", - "RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS declares the " - "*_contributions_desired input columns in release_input_coverage.py", + "IRS SOI / W-2 retirement-contribution aggregate the retired pipeline " + "calibrated (PR #496/#554) but which it and populace both moved to the input " + "side: modeled as imputed pre-limit contribution INPUT columns, not " + "reweighting targets.", + "POST_REFERENCE_ECPS_REQUIRED_INPUTS declares the *_contributions_desired " + "input columns in release_input_coverage.py", + _fence( + origin=( + "us-data PR #496 (IRA targets) and PR #554 (401k/SE " + "pension targets), converged at PR #1125" + ), + purpose=( + 'ensure income-contribution consistency ("$0-wage PUF clones get $0 ' + 'in 401(k) contributions", PR #554); target the ALD deduction not ' + 'raw contributions because "the variable flows directly into the ALD ' + "with no deductibility logic in policyengine-us, so the target must " + 'match the deduction, not total contributions" (loss.py)' + ), + verdict_basis=( + "architecture retires it: us-data PR #1125 and populace both govern " + "retirement contributions as imputed pre-limit INPUT columns " + "(*_contributions_desired, POST_REFERENCE_ECPS_REQUIRED_INPUTS in the " + "input-coverage contract) with PolicyEngine-US applying statutory " + "limits — calibrating them here would double-govern a quantity the " + "input-coverage gate owns. The feed carries the amounts, so this is " + "not a source gap." + ), + ), ) _SNAP_PERSONS_EXCLUSION = ( "not_modeled", - "USDA FNS SNAP average-monthly PERSONS. The SNAP assistance unit is often a " - "subset of the SPM unit (FY2024 FNS persons-per-household 1.88 vs 2.82 " - "simulated members per taker unit), so a person indicator overcounts FNS " - "participants by ~50%. The household caseload " - "(usda_snap.national/state_average_monthly_households, compiled) is the " - "calibrated count; persons stay unmapped until sub-unit participation is " - "modeled.", + "USDA FNS SNAP average-monthly PERSONS. A person indicator overcounts FNS " + "participants ~50% (FY2024 persons-per-household 1.88 vs 2.82 simulated " + "members); the household caseload is the calibrated count.", "INDICATOR_LEDGER_TARGETS average_monthly_households comment + " "test_snap_person_caseload_fact_is_not_compiled", + _fence( + origin=( + "not a us-data calibration target as a persons row: us-data " + "calibrates SNAP dollars (CBO_PROGRAMS 'snap'), and the household " + "caseload (usda_snap.*_average_monthly_households) is compiled." + ), + purpose="n/a — the household caseload is the model counterpart, not persons.", + verdict_basis=( + "not_modeled: PolicyEngine-US does not model sub-unit SNAP " + "participation, so a person indicator overcounts ~50% vs the compiled " + "household caseload." + ), + ), ) -# (classification, reason, evidence) keyed by exact family id. -_FAMILY_EXCLUSIONS: dict[str, tuple[str, str, str]] = { +# (classification, reason, evidence, fence) keyed by exact family id. +_FAMILY_EXCLUSIONS: dict[str, tuple[str, str, str, dict[str, str]]] = { + "bea_nipa.personal_interest_income": ( + "macro_control_total", + "BEA NIPA personal interest income. Briefly a direct target (PR #994), " + "then explicitly declined (PR #1059): the NIPA total includes imputed " + "interest and trust flows that are not a close microdata concept.", + "irs_soi.historic_table_2 (compiled) carries the tax-return interest; " + "sample fact bea_nipa.cy2023.personal_interest_income.a064rc.amount", + _fence( + origin="us-data PR #994 (added) -> PR #1059 (removed)", + purpose=( + '"BEA personal interest/dividends include imputed interest, ' + "pension-plan dividends, and trust flows, so those macro totals " + "should not directly calibrate tax/CPS interest and dividend " + 'variables" (loss.py:67-70, added by PR #1059)' + ), + verdict_basis=( + "us-data itself declined it as a direct target (PR #1059): a " + "macro benchmark, not a close microdata concept. Tax-return " + "interest is calibrated from IRS SOI + CBO (compiled)." + ), + ), + ), + "bea_nipa.personal_dividend_income": ( + "macro_control_total", + "BEA NIPA personal dividend income. Briefly a direct target (PR #994), " + "then explicitly declined (PR #1059): the NIPA total includes dividends " + "received through pension funds and private trusts.", + "cbo.revenue_projection (qualified dividends) + irs_soi (compiled); " + "sample fact bea_nipa.cy2023.personal_dividend_income.b703rc.amount", + _fence( + origin="us-data PR #994 (added) -> PR #1059 (removed)", + purpose=( + '"NIPA includes dividends received through pension funds and ' + "private trusts, so this is a macro benchmark rather than a pure " + 'tax concept" (deleted etl_national_targets.py note, PR #1059; ' + "FRED B703RC1A027NBEA)" + ), + verdict_basis=( + "us-data itself declined it (PR #1059) as non-comparable; " + "dividends are calibrated from the CBO qualified-dividend series " + "(cbo.revenue_projection) and SOI (compiled)." + ), + ), + ), + "bea_regional.state_wages_salaries": ( + "deferred", + "BEA regional state wages — a real us-data target (PR #1034), but us-data " + "residence-adjusts SAINC4 line-50 place-of-work wages and scales them to " + "the national total, an adjustment the feed's raw place-of-work facts " + "cannot reproduce. The national wage aggregate is compiled.", + "compiled national aggregate bea_nipa.total_wages_salaries; feed fact " + "bea_regional.cy2023.state_wages_salaries.dc.amount = $96.8B is " + "place-of-work, far above DC residence wages", + _fence( + origin="us-data PR #1034 (Fixes #1033)", + purpose=( + '"residence-adjust state wages using SAINC4 line 42 before ' + "scaling to the national NIPA wages target ... so the national " + 'and state wage controls bind" (PR #1034); the state distribution ' + "of the nonfiler-inclusive wage universe" + ), + verdict_basis=( + "deferred: the feed carries raw place-of-work SAINC4 line-50 state " + "wages, but us-data applies a place-of-work -> residence " + "adjustment (line 42 apportioned by wage share) plus national " + "scaling in bea_regional.py that is not reproducible from the " + "feed. The national wage aggregate (bea_nipa.total_wages_salaries) " + "is compiled; the state distribution is deferred pending " + "residence-adjusted state wages in the ledger feed." + ), + ), + ), "cbo.revenues": ( "macro_control_total", - "CBO total federal revenue line (fiscal-year budget aggregate). The " - "household-linear CBO surface populace calibrates is the income-by-" - "source projection (cbo.revenue_projection, compiled as cbo:5); a total " - "revenue outturn is an aggregate, not a per-record target.", + "CBO total federal revenue line (fiscal-year budget aggregate) us-data " + "did not calibrate to. The household-linear CBO surface it used is the " + "income-by-source projection (cbo.revenue_projection, compiled).", "compiled sibling cbo.revenue_projection; sample fact " "cbo.fy2023.revenues.individual_income_taxes.actual_amount", + _fence( + origin=( + "not a us-data calibration target: us-data calibrates the CBO " + "income-by-source projections (CBO_INCOME_BY_SOURCE_TARGETS), not " + "the total-revenue outturn." + ), + purpose="n/a — an aggregate outturn, never a per-record target.", + verdict_basis=( + "no fence to rebuild; component income is compiled via " + "cbo.revenue_projection." + ), + ), ), "census.popproj2023": ( "superseded", - "Census Bureau population PROJECTIONS (forward vintage popproj2023). " - "Population is calibrated to the OBSERVED Census resident-population " - "estimates (census_pep.v2024 and national_resident_population_age, " - "compiled as census_pep:936); the projection series is a forward-looking " - "alternative, not the observed administrative count.", + "Census Bureau population PROJECTIONS (forward vintage). us-data " + "calibrated the OBSERVED resident population, compiled via census_pep.", "compiled sibling census_pep.v2024; sample fact " "census.popproj2023.cy2023.national_population.age_0.population", + _fence( + origin=( + "not a us-data calibration target: us-data calibrates observed " + "Census population (census_pep), not the projection series." + ), + purpose="n/a — a forward projection, never the observed control.", + verdict_basis=( + "superseded by compiled census_pep.v2024 / " + "national_resident_population_age (observed resident population)." + ), + ), ), "census_acs.acs1_2023": ( "survey_derived", - "American Community Survey 1-year estimates. ACS is a household SAMPLE " - "SURVEY, not an administrative universe, so its aggregates are excluded " - "by principle. ACS age detail is used only at congressional-district " - "grain when include_congressional_district_targets is enabled (off for " - "the national release).", + "American Community Survey 1-year estimates — a household SAMPLE SURVEY, " + "not an administrative universe. Used at congressional-district grain " + "only, off for the national release.", "census_acs facts return None unless include_congressional_district_" "targets in fiscal_targets._reference_from_ledger_fact; sample fact " "census_acs.acs1_2023.b01001.female_age.01.age_15_to_17.female_population", + _fence( + origin=( + "not a national us-data calibration target: ACS feeds regional/CD " + "H5 targets in us-data, off for the national build." + ), + purpose="n/a nationally — survey-derived CD detail, not a national fence.", + verdict_basis=( + "off_by_default + survey_derived: excluded by principle (ACS is a " + "sample survey) and only relevant at CD grain " + "(include_congressional_district_targets, off here)." + ), + ), ), "cms_nhe.medicaid_title_xix_expenditures": ( "non_linear", - "CMS National Health Expenditure Medicaid Title XIX spending. Already " - "declared calibration_role=validation_only in DIRECT_LEDGER_TARGETS: " - "PolicyEngine-US allocates Medicaid spending from state totals through " - "person_weight-dependent denominators, so reweighting recomputes " - "per-person costs and this is not a linear calibration row.", + "CMS NHE Medicaid Title XIX spending — a real us-data target (PR #292) " + "whose linear-target assumption PolicyEngine-US retired: medicaid cost is " + "now a person_weight-dependent allocation, so this is not a linear " + "calibration row. Kept as calibration_role=validation_only.", "DIRECT_LEDGER_TARGETS ('cms_nhe','expenditure_amount','medicaid_title_" "xix') metadata calibration_role=validation_only", - ), - "federal_reserve_z1.households_nonprofits_balance_sheet": ( - "macro_control_total", - "Federal Reserve Financial Accounts (Z.1) households & nonprofits " - "net-worth aggregate. A macro balance-sheet total; household net worth " - "is imputed and calibrated from SCF micro on the input side " - "(scf_wealth source stage), not from this national aggregate.", - "scf_wealth.py source stage supplies the net-worth micro; sample fact " - "federal_reserve_z1.cy2023.households_nonprofits_balance_sheet.net_worth" - ".fl152090005.amount_outstanding", - ), - "hhs_acf_liheap.national_profile": ( - "deferred", - "HHS LIHEAP national profile (households served / funds). PolicyEngine-US " - "does not yet expose a LIHEAP receipt outcome to fit this count to a " - "per-household model counterpart, so it is deferred rather than dropped; " - "retained as a ledger reference fact.", - "no LIHEAP target_role in INDICATOR_LEDGER_TARGETS/DIRECT_LEDGER_TARGETS; " - "sample fact hhs_acf_liheap.fy2023.national_profile.state_programs." - "households_served", + _fence( + origin="us-data PR #292 (medicaid spending target)", + purpose=( + "keep weighted Medicaid outlays aligned to the CMS national total " + "(nation/hhs/medicaid_spending). No discoverable prose rationale " + "in PR #292 (only a value/source note); the mechanical purpose is " + "the aggregate-spending control." + ), + verdict_basis=( + "architecture retires it: PolicyEngine-US PR #1138 made " + "medicaid_cost_if_enrolled an SLCSP allocation from state totals " + "normalized against enrollees — person_weight-dependent, so " + "reweighting recomputes per-person costs and it is not a linear " + "calibration row. populace keeps it calibration_role=" + "validation_only in DIRECT_LEDGER_TARGETS." + ), + ), ), "hhs_acf_tanf.average_monthly_families": ( "deferred", "HHS ACF TANF average-monthly family caseload. TANF is calibrated on " - "BENEFIT DOLLARS (hhs_acf_tanf.cash_assistance, compiled as " - "hhs_acf_tanf:30); the caseload COUNT is not yet wired to a TANF-receipt " - "indicator (the SNAP-household caseload analog for TANF), so it is " - "deferred.", - "compiled sibling hhs_acf_tanf.cash_assistance carries the dollar " - "targets; sample fact " + "BENEFIT DOLLARS (compiled); the caseload COUNT has no wired TANF-receipt " + "indicator yet.", + "compiled sibling hhs_acf_tanf.cash_assistance; sample fact " "hhs_acf_tanf.fy2024.average_monthly_families.us.us_total.total_families", + _fence( + origin=( + "not a us-data calibration target: us-data calibrates TANF dollars " + '(HARD_CODED_TOTALS "tanf"), compiled via cash_assistance; the ' + "caseload count is not a us-data target." + ), + purpose="n/a — TANF dollars are the target, not the caseload count.", + verdict_basis=( + "deferred: no TANF-receipt indicator is wired (unlike the SNAP / " + "LIHEAP household caseloads); the dollar target is compiled." + ), + ), ), "hhs_acf_tanf.average_monthly_recipients": ( "deferred", - "HHS ACF TANF average-monthly recipient caseload. As with the family " - "caseload, TANF is calibrated on benefit dollars " - "(hhs_acf_tanf.cash_assistance, compiled); the recipient COUNT has no " - "wired TANF-receipt indicator yet and is deferred.", + "HHS ACF TANF average-monthly recipient caseload. As with families, TANF " + "is calibrated on benefit dollars (compiled); the recipient COUNT has no " + "wired TANF-receipt indicator.", "compiled sibling hhs_acf_tanf.cash_assistance; sample fact " "hhs_acf_tanf.fy2024.average_monthly_recipients.us.us_total." "total_recipients", + _fence( + origin=( + "not a us-data calibration target: us-data calibrates TANF " + "dollars (compiled); the recipient count is not a us-data target." + ), + purpose="n/a — TANF dollars are the target, not the recipient count.", + verdict_basis=( + "deferred: no TANF-receipt indicator wired; the dollar target is " + "compiled via cash_assistance." + ), + ), ), "irs_soi.congressional_district_2022": ( "off_by_default", "IRS SOI congressional-district table. CD-level targets are opt-in " - "(include_congressional_district_targets=False for the national " - "release); the national and state SOI surfaces are compiled instead. " - "Enabling CD targets compiles these — they are excluded for the national " - "build by design, not dropped.", + "(include_congressional_district_targets=False for the national release); " + "the national and state SOI surfaces are compiled instead.", "test_soi_congressional_district_targets_are_opt_in + the " "include_congressional_district_targets gate in " "fiscal_targets._soi_reference_from_fact", + _fence( + origin=( + "us-data CD targets serve the regional/local H5 outputs " + "(build_outputs/target_universe.py), not the national build." + ), + purpose=( + "distributional shape within a congressional district for local " + "H5 outputs." + ), + verdict_basis=( + "off_by_default: CD targets are opt-in for the national release; " + "enabling include_congressional_district_targets compiles them. " + "The national and state SOI surfaces are compiled." + ), + ), ), "irs_soi.form_w2_401k_elective_deferrals": _RETIREMENT_CONTRIBUTION_EXCLUSION, "irs_soi.form_w2_designated_roth_401k_contributions": ( @@ -234,55 +414,107 @@ "irs_soi.traditional_ira_contributions": _RETIREMENT_CONTRIBUTION_EXCLUSION, "irs_soi.form_w2_social_security_tips": ( "not_modeled", - "IRS W-2 Social Security tip aggregates. Already declared in " - "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS: current US support does not " - "materialize a positive tip_income source column, so the W-2 tip return " - "counts need the SIPP/ORG tip source stage wired before calibration.", + "IRS W-2 Social Security tips — a real us-data target (PR #220) the model " + "cannot yet satisfy: PolicyEngine-US produces a structural zero for " + "tip_income in the populace base microdata (no tip source column), so the " + "target is unsatisfiable until the tip-imputation source stage is ported.", "US_FISCAL_TARGET_SUPPORT_EXCLUSIONS entry " "irs_soi.ty2023.form_w2_social_security_tips.box_7_social_security_tips." "return_count", + _fence( + origin="us-data PR #220 (Impute tips, fixes #215)", + purpose=( + "\"Proposals such as 'No Tax on Tips' require a clean tip_income " + "field distinct from regular wages ... Tipped workers skew " + "lower-income; omitting tips biases poverty, EITC/CTC, and " + 'payroll-tax results" (issue #215)' + ), + verdict_basis=( + "deferred (not source-absent — the feed carries a tip amount): " + "PolicyEngine-US yields a structural zero for tip_income in the " + "populace base microdata (US_FISCAL_TARGET_SUPPORT_EXCLUSIONS), so " + "the target is unsatisfiable until the SIPP/ORG tip-imputation " + "source stage (us-data #220) is ported. Wiring it now would ship a " + "0-vs-target gap, not a fit." + ), + ), ), "kff.marketplace_effectuated_enrollment": ( "superseded", - "Kaiser Family Foundation state marketplace effectuated-enrollment " - "compilation. ACA marketplace enrollment is calibrated from the primary " - "CMS administrative source (cms_aca.oep2024, compiled as cms_aca:102); " - "KFF is a secondary aggregator of the same underlying CMS data, excluded " - "to avoid a duplicate target.", + "Kaiser Family Foundation state marketplace enrollment — a secondary " + "aggregator of the CMS data us-data calibrated. ACA enrollment is fit " + "from the primary CMS source, compiled.", "compiled sibling cms_aca.oep2024; sample fact " "kff.marketplace_effectuated_enrollment.2024.state.us." "total_effectuated_marketplace_enrollment", + _fence( + origin=( + "not a us-data calibration target: us-data calibrates ACA " + "enrollment from CMS (nation/gov/aca_enrollment), not KFF." + ), + purpose="n/a — a secondary compilation of the same CMS data.", + verdict_basis=( + "superseded by compiled cms_aca.oep2024 (the primary CMS " + "administrative source); avoids a duplicate target." + ), + ), ), "usda_snap.national_average_monthly_persons": _SNAP_PERSONS_EXCLUSION, "usda_snap.state_average_monthly_persons": _SNAP_PERSONS_EXCLUSION, } -# Administrative target families the retired us-data pipeline calibrated to for -# which the pinned feed carries NO ledger fact — source-absent by the feed. -# (family id, reason, evidence) with classification source_absent. -_SOURCE_ABSENT_US_DATA_FAMILIES: dict[str, tuple[str, str]] = { +# us-data targets the pinned feed carries NO ledger fact for. (reason, evidence, +# fence) with classification source_absent. +_SOURCE_ABSENT_US_DATA_FAMILIES: dict[str, tuple[str, str, dict[str, str]]] = { "bls.consumer_expenditure": ( - "BLS Consumer Expenditure Survey aggregates targeted by the retired " - "us-data pipeline (nation/bls/ce). The pinned consumer feed carries no " - "BLS source fact, so there is no ledger-shaped fact to compile; " - "source-absent pending a Ledger BLS CE ingest.", - "retired us-data pipeline (archived) references nation/bls/ce; feed " - "source families carry no 'bls' source", + "BLS Consumer Expenditure childcare-expense target the retired pipeline " + "calibrated (loss.py BLS_CE_TOTALS childcare_expenses = $63.09B). The " + "pinned feed carries no BLS source fact.", + "retired us-data pipeline (archived) loss.py BLS_CE_TOTALS + " + "nation/bls/ce; feed source families carry no 'bls' source", + _fence( + origin="us-data loss.py BLS_CE_TOTALS (BLS CE LABSTAT)", + purpose=( + "anchor modeled childcare expenses to the BLS Consumer " + "Expenditure aggregate ($63.09B, 2024)." + ), + verdict_basis=( + "source-absent: the pinned feed carries no BLS CE ledger fact, so " + "there is nothing to compile; deferred pending a Ledger BLS CE " + "ingest." + ), + ), ), "wic.national_summary": ( - "USDA WIC national annual summary targeted by the retired us-data " - "pipeline (WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE in etl_national_targets). " - "The pinned feed carries no WIC source fact; source-absent. WIC receipt " - "is modeled via the would_claim_wic take-up input, not a target.", + "USDA WIC national annual summary the retired pipeline calibrated " + "(WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE). The pinned feed carries no WIC " + "fact; WIC receipt is modeled via the would_claim_wic input.", "retired us-data pipeline (archived) db/etl_national_targets.py " "WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE; feed carries no 'wic' source", + _fence( + origin=( + "us-data db/etl_national_targets.py WIC_NATIONAL_ANNUAL_SUMMARY_SOURCE" + ), + purpose="anchor WIC participation to the USDA national summary.", + verdict_basis=( + "source-absent: no WIC ledger fact in the feed. WIC receipt is " + "modeled via the would_claim_wic take-up input, not a target here." + ), + ), ), "hud.housing_assistance": ( - "HUD housing-assistance aggregates targeted by the retired us-data " - "pipeline (db/etl_housing_assistance.py). The pinned feed carries no HUD " - "source fact; source-absent pending a Ledger HUD ingest.", - "retired us-data pipeline (archived) db/etl_housing_assistance.py; " - "feed carries no 'hud' source", + "HUD housing-assistance aggregates the retired pipeline calibrated " + "(db/etl_housing_assistance.py). The pinned feed carries no HUD fact.", + "retired us-data pipeline (archived) db/etl_housing_assistance.py; feed " + "carries no 'hud' source", + _fence( + origin="us-data db/etl_housing_assistance.py", + purpose="anchor modeled housing assistance to HUD admin totals.", + verdict_basis=( + "source-absent: no HUD ledger fact in the feed; deferred pending a " + "Ledger HUD ingest." + ), + ), ), } @@ -318,17 +550,18 @@ def _compiled_families(facts: list[dict]) -> set[str]: } -def _exclusion_for(family: str) -> tuple[str, str, str]: +def _exclusion_for(family: str) -> tuple[str, str, str, dict[str, str]]: if family in _FAMILY_EXCLUSIONS: return _FAMILY_EXCLUSIONS[family] namespace = family.split(".", 1)[0] if namespace in _NAMESPACE_EXCLUSIONS: return _NAMESPACE_EXCLUSIONS[namespace] raise SystemExit( - f"Feed family {family!r} neither compiles nor has a declared reviewed " - "exclusion. Add an entry to _FAMILY_EXCLUSIONS or _NAMESPACE_EXCLUSIONS " - "in tools/build_us_target_parity_manifest.py naming its evidence — a new " - "administrative family must never be silently ignored." + f"Feed family {family!r} neither compiles nor has a declared, fenced " + "reviewed exclusion. Add an entry to _FAMILY_EXCLUSIONS or " + "_NAMESPACE_EXCLUSIONS in tools/build_us_target_parity_manifest.py with a " + "fence {origin, purpose, verdict_basis} — a category label alone is not a " + "sufficient reason to drop a us-data-era calibration target." ) @@ -343,16 +576,15 @@ def build_manifest( if family in compiled: families[family] = {"status": COMPILED_STATUS} else: - classification, reason, evidence = _exclusion_for(family) + classification, reason, evidence, fence = _exclusion_for(family) families[family] = { "status": REVIEWED_EXCLUSION_STATUS, "classification": classification, "reason": reason, "evidence": evidence, + "fence": fence, } - # Compiled families with no feed fact should not exist (every compiled spec - # traces to a feed fact); guard against a mapping that invents a family. invented = sorted(compiled - set(feed_counts)) if invented: raise SystemExit( @@ -360,12 +592,15 @@ def build_manifest( "The family-id derivation or the wiring is inconsistent." ) - for family, (reason, evidence) in sorted(_SOURCE_ABSENT_US_DATA_FAMILIES.items()): + for family, (reason, evidence, fence) in sorted( + _SOURCE_ABSENT_US_DATA_FAMILIES.items() + ): families[family] = { "status": REVIEWED_EXCLUSION_STATUS, "classification": SOURCE_ABSENT_CLASSIFICATION, "reason": reason, "evidence": evidence, + "fence": fence, } n_compiled = sum(1 for e in families.values() if e["status"] == COMPILED_STATUS) @@ -381,12 +616,18 @@ def build_manifest( "apply_us_medicaid_enrollment_substitutions" ), "us_data_source": ( - "retired us-data pipeline (archived): db/etl_national_targets.py, " - "db/etl_*.py, utils/national_target_parity.py" + "retired us-data pipeline (archived): utils/loss.py (eCPS loss " + "matrix), db/etl_national_targets.py, db/etl_*.py, " + "utils/national_target_parity.py" ), "family_granularity": ( "namespace.concept of the ledger source_record_id (us_target_family_id)" ), + "governing_rule": ( + "us-data-targeted => compiled unless source-absent; every " + "reviewed exclusion carries a Chesterton's-fence " + "{origin, purpose, verdict_basis}" + ), "compiled_families": str(n_compiled), "reviewed_exclusions": str(n_reviewed), },