Full eCPS input coverage: 158 required columns, 8 evidence-backed exclusions

experiments/build_j_recert/buildj_base.sh

@@ -1,11 +1,9 @@
 #!/bin/bash
-# Build J STEP 1 — base pool rebuild on main's stages (populace#368 re-certification).
-# Replicates Build F's base command EXACTLY on origin/main's base builder
-# (byte-identical to Build F's tools/build_us_puf_support_base.py) to (a) satisfy the
-# #368 "base pool rebuild" step and (b) EMPIRICALLY confirm determinism. The SNAP
-# (#350/#352/#353) and SCF-wealth (#373) enrichments are RELEASE-TIME source stages
-# (us_runtime/*), NOT base-builder stages, so the base sha is expected == Build F's
-# 18833fb6 (documented finding vs the task's "new base sha expected" premise).
+# Build J STEP 1 — rebuild the current source-staged base pool.
+# The branch now restores CPS/ACS housing inputs in the reusable base builder,
+# so Build F's pre-housing base and any pre-prior-year-income base are stale.
+# Cache reuse is allowed only when every newly restored persisted leaf carries
+# signal.
 #
 # Compute discipline: ~12 min (Build F was 12m14s), one chunk (<30 min). Detached via
 # setsid+caffeinate; pidfile + real rc + append log + memory-pressure sampler.
@@ -27,28 +25,71 @@ TS=$(date -u +%Y%m%dT%H%M%SZ)
 PLOG="$LOGDIR/pressure_base_$TS.log"
 mkdir -p "$LOGDIR" "$OUTDIR"
 say() { echo "[$(date -u +%FT%TZ)] $1" | tee -a "$LOG"; }
+base_has_restored_signal() {
+  .venv/bin/python -c '
+import sys
+import pandas as pd
+
+required = {
+    "person": (
+        "pre_subsidy_rent",
+        "self_employment_income_last_year",
+        "previous_year_income_available",
+        "salt_refund_income",
+        "takes_up_medicare_if_eligible",
+        "workers_compensation",
+        "would_claim_wic",
+    ),
+    "spm_unit": (
+        "receives_housing_assistance",
+        "takes_up_housing_assistance_if_eligible",
+        "spm_unit_tenure_type",
+        "spm_unit_energy_subsidy",
+    ),
+    "household": ("tenure_type",),
+}
+try:
+    with pd.HDFStore(sys.argv[1], mode="r") as store:
+        tables = {
+            entity: store.select(entity, columns=list(names))
+            for entity, names in required.items()
+        }
+except (KeyError, TypeError, ValueError):
+    raise SystemExit(1)
+ok = all(
+    name in tables[entity] and tables[entity][name].nunique(dropna=False) > 1
+    for entity, names in required.items()
+    for name in names
+)
+raise SystemExit(0 if ok else 1)
+' "$1"
+}
 rm -f "$LOGDIR/base.rc"
 
 cd "$WT" || { say "FATAL cannot cd $WT"; echo 2 > "$LOGDIR/base.rc"; exit 2; }
 source .venv/bin/activate 2>/dev/null
 
 # ---- integrity preflight on inputs (reproduce Build F's exact base) ----
 chk() { local got exp; got=$(shasum -a 256 "$1" | cut -c1-16); [ "$got" = "$2" ] || { say "FATAL sha mismatch $1: $got != $2"; echo 2 > "$LOGDIR/base.rc"; exit 2; }; }
-for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do
+for f in "$USD/census_cps_2024.h5" "$USD/census_cps_2023.h5" "$USD/census_cps_2022.h5" "$USD/puf_2024.h5" "$USD/acs_2022.h5" "$AGING_FACTS" "$CDX" "$BLADDER"; do
   [ -f "$f" ] || { say "FATAL missing input $f"; echo 2 > "$LOGDIR/base.rc"; exit 2; }
 done
 chk "$AGING_FACTS" a5d34d4aad325d8c
 chk "$CDX" 383a666631aafd4f
 chk "$BLADDER" 7ba39b959068181b
+chk "$USD/acs_2022.h5" 0b319b496f19a691
 
 SHORT=$(git rev-parse --short HEAD)
 PEUS=$(.venv/bin/python -c "from importlib.metadata import version; print(version('policyengine-us'))" 2>/dev/null)
 say "BUILD J BASE START commit=$SHORT pe-us=$PEUS pid=$$ pressure_log=$PLOG"
-say "  inputs: asec 2024/2023/2022 + puf_2024; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32"
+say "  inputs: asec 2024/2023/2022 + puf_2024 + acs_2022; aging-facts a5d34d4a; cdx 383a6666; bladder 7ba39b95; seed 0 n-est 32"
 
-if [ -f "$BASE" ]; then
-  say "BASE: already present at $BASE — verifying sha only (delete to force rebuild)"
+if [ -f "$BASE" ] && base_has_restored_signal "$BASE"; then
+  say "BASE: already present with nondefault restored input surface at $BASE — reusing"
 else
+  if [ -f "$BASE" ]; then
+    say "BASE: cached artifact is stale or restored-input-default-only — rebuilding in place"
+  fi
   # ---- memory-pressure sampler (self-terminates on python exit; SAMPLE ONLY, no kill) ----
   ( echo "ts_utc,free_pct,swap_used_mb,py_rss_mb,py_pid"
     while :; do
@@ -62,12 +103,13 @@ else
     done ) >> "$PLOG" 2>&1 &
   SAMPLER=$!
 
-  say "BASE: rebuilding (3-year ASEC pool 2024+2023+2022, 2x PUF clone, spec-less equal thirds)"
+  say "BASE: rebuilding (3-year ASEC pool 2024+2023+2022, pinned ACS 2022 rent donor, 2x PUF clone, spec-less equal thirds)"
   .venv/bin/python tools/build_us_puf_support_base.py \
     --asec-h5 2024="$USD/census_cps_2024.h5" \
     --asec-h5 2023="$USD/census_cps_2023.h5" \
     --asec-h5 2022="$USD/census_cps_2022.h5" \
     --puf-h5 "$USD/puf_2024.h5" \
+    --acs-h5 "$USD/acs_2022.h5" \
     --target-year 2024 \
     --seed 0 --n-estimators 32 \
     --ledger-facts "$AGING_FACTS" \
@@ -83,13 +125,21 @@ else
   if [ $rc -ne 0 ]; then echo "$rc" > "$LOGDIR/base.rc"; say "BASE FAILED rc=$rc — diagnose in base_j.log"; exit "$rc"; fi
 fi
 
+if ! base_has_restored_signal "$BASE"; then
+  echo 2 > "$LOGDIR/base.rc"
+  say "BASE FAILED: output does not persist all restored nondefault inputs"
+  exit 2
+fi
+
 BASE_SHA=$(shasum -a 256 "$BASE" | cut -d' ' -f1)
 echo "$BASE_SHA" > "$LOGDIR/base.sha"
 say "BASE sha: $BASE_SHA"
 if [ "$BASE_SHA" = "18833fb68e60ee74461608d81a5c5ab7d52435e17026d9e3b062d9de18d6871f" ]; then
-  say "BASE sha CONFIRMS Build F 18833fb6 — base pool unchanged (assets/SNAP are release-time source stages, not base-builder stages)"
+  echo 2 > "$LOGDIR/base.rc"
+  say "BASE FAILED: Build F's pre-housing SHA was reused"
+  exit 2
 else
-  say "BASE sha DIFFERS from 18833fb6 — investigate before proceeding (base builder or deps changed the pool)"
+  say "BASE sha differs from pre-housing Build F as required"
 fi
 echo "0" > "$LOGDIR/base.rc"
 say "BUILD J BASE DONE rc=0 sha=$BASE_SHA"

packages/populace-build/src/populace/build/source_manifest.py

@@ -50,26 +50,54 @@
         "compute_ratio",
         "declare_income_reference_offset",
         "derive",
+        "derive_childcare_inputs",
+        "derive_child_support_inputs",
+        "derive_disability_benefits",
+        "derive_energy_subsidy",
+        "derive_education_inputs",
         "derive_eligibility_inputs",
         "derive_hours_worked",
+        "derive_housing_tenure_inputs",
         "derive_immigration_status",
+        "derive_medicare_take_up",
+        "derive_other_health_insurance_premiums",
+        "derive_prior_year_income",
         "derive_snap_abawd_discretionary_exemption",
         "derive_snap_take_up",
         "derive_puf_policyengine_variables",
         "derive_mortgage_balance_hints",
         "derive_pregnancy",
+        "derive_relationship_inputs",
+        "derive_retirement_distributions",
+        "derive_retirement_contributions",
+        "derive_workers_compensation",
+        "derive_weeks_unemployed",
+        "derive_wic_claim",
         "disaggregate_aggregate_records",
         "fit_labor_market_models",
         "fit_tip_income_model",
+        "fit_weighted_acs_rent_qrf",
         "fit_vehicle_model",
         "fit_weighted_imputer",
         "fit_weighted_qrf",
         "fold_into",
         "head_carry",
         "join",
+        "impute_retirement_contributions_to_puf_support",
+        "impute_childcare_to_puf_support",
+        "impute_child_support_to_puf_support",
+        "impute_disability_benefits_to_puf_support",
+        "impute_energy_subsidy_to_puf_support",
+        "impute_housing_assistance_to_puf_support",
+        "impute_other_health_insurance_premiums_to_puf_support",
+        "impute_prior_year_income_to_puf_support",
+        "impute_retirement_distributions_to_puf_support",
+        "impute_workers_compensation_to_puf_support",
+        "impute_weeks_unemployed_to_puf_support",
         "map_columns",
         "read_table",
         "read_tables",
+        "read_acs_rent_donor",
         "replace_sentinels",
         "split_component_by_share",
         "support_clip",