US support-mix bake-off: CPS years vs ACS rows vs location clones at fixed budgets

changelog.d/us-support-mix-bakeoff.added.md

@@ -0,0 +1 @@
+Add a support-mix bake-off (tools/bakeoff_us_support_mix.py, us_runtime/support_mix.py): CPS years vs ACS rows vs CPS location clones at fixed household budgets, calibrated with the production solver and scored on the hash-split holdout; memo in docs/us-support-mix-bakeoff.md.

experiments/us-support-mix-bakeoff-20260930/README.md

@@ -0,0 +1,32 @@
+# Support-mix bake-off receipts (2026-09-30)
+
+Outputs of `tools/bakeoff_us_support_mix.py report` over the 60 arm receipts
+(30 arms × national and local products), plus the compile and differential
+receipts. [`docs/us-support-mix-bakeoff.md`](../../docs/us-support-mix-bakeoff.md)
+gives the design, verdict and caveats.
+
+| File | What |
+|---|---|
+| `dimensions_by_arm.csv` | One row per arm and product: rows, distinct-household ESS, held-out error by dimension and level, ACS-native error by level |
+| `headline.md` | The same, as markdown tables by product |
+| `arms.csv` | Per-arm counts, ESS variants, training loss, solve time, peak RSS, report-only SPM rates |
+| `holdout_by_dimension_level.csv` | Held-out error by product, arm, dimension group, dimension and level (unsupported targets score as misses) |
+| `holdout_by_dimension_level_common_support.csv` | The same over targets every arm supports |
+| `holdout_unsupported.csv` | Held-out targets no row in an arm can move |
+| `replicate_spread.csv` | SD across three disjoint ACS draws (300k arms) |
+| `replicate_mean.csv` | Mean across the same three draws |
+| `acs_native_by_level_measure.csv` | ACS-native error by product, arm, level and measure |
+| `compile.json` | Registry compile: 32,842 specs, 419 concepts, split roles by level |
+| `diffcheck.json` | Production per-target columns vs concept × mask on 4,000 households per source: max scaled difference 0.0 |
+| `acs_truth.json` | ACS 2024 1-year summary-file cells and their SHA-256 |
+| `partcheck_cps.json`, `partcheck_acs.json` | Stored concept parts against a fresh materialization, under the final concept specs, of 200 sampled households per part: max scaled difference 5.8e-8 |
+| `spm_flags.json` | ACS group-quarters share and the SPM zero-adult records the support leaves out (`spm-flags`) |
+| `materialize/` | Per-part engine-pass receipts (households, wall time, peak RSS) and the measure list |
+| `asec_quality.py`, `asec_quality.json` | CPS year response-quality check (script and its output) |
+| `memo_tables.py` | Builds the memo's results tables from these files |
+
+The folder holds no per-arm receipts (10–15 MB each, with every held-out
+target's estimate) and no saved weights. Both stayed in the run's work
+directory, which no longer exists. The `materialize/` receipts predate the
+source-h5, concept-spec and null-fill-summary digests the memo describes; on
+2026-09-30 their numerical fields matched the work copies'.

experiments/us-support-mix-bakeoff-20260930/acs_truth.json

@@ -0,0 +1,71 @@
+{
+ "cells": {
+  "households": [
+   "B25003_E001"
+  ],
+  "owner": [
+   "B25003_E002"
+  ],
+  "persons_0_17": [
+   "B01001_E003",
+   "B01001_E004",
+   "B01001_E005",
+   "B01001_E006",
+   "B01001_E027",
+   "B01001_E028",
+   "B01001_E029",
+   "B01001_E030"
+  ],
+  "persons_65p": [
+   "B01001_E020",
+   "B01001_E021",
+   "B01001_E022",
+   "B01001_E023",
+   "B01001_E024",
+   "B01001_E025",
+   "B01001_E044",
+   "B01001_E045",
+   "B01001_E046",
+   "B01001_E047",
+   "B01001_E048",
+   "B01001_E049"
+  ],
+  "persons_in_households": [
+   "B25008_E001"
+  ],
+  "persons_male": [
+   "B01001_E002"
+  ],
+  "persons_total": [
+   "B01003_E001"
+  ],
+  "real_estate_taxes_aggregate": [
+   "B25090_E001"
+  ],
+  "rent_contract_monthly_aggregate": [
+   "B25060_E001"
+  ],
+  "rent_gross_monthly_aggregate": [
+   "B25065_E001"
+  ],
+  "renter": [
+   "B25003_E003"
+  ]
+ },
+ "files_sha256": {
+  "acsdt1y2024-b01001.dat": "84db6a3a8eda54d882b9a5e6a5bb48864fcbb573a262127ed026ae5fb6d6953e",
+  "acsdt1y2024-b01003.dat": "3f04d8c2ad77dab54085f014b795c33c19463eede4539e110247bb329d1025fd",
+  "acsdt1y2024-b25003.dat": "944b9d7aa807cc32c50a50b37af0afac50db914f02dc68bafa628e4227a496ad",
+  "acsdt1y2024-b25008.dat": "6e8e40d7679c47b90df6504c449bde6b28c4384c329b68af0dd436c6943418ca",
+  "acsdt1y2024-b25060.dat": "aa6d70941599febec9c35e60d8692df6bbc2c01146d2d18d912d9bd24dbfd457",
+  "acsdt1y2024-b25065.dat": "e17f8ebae8941b0f7466d0e3cdad5cca6425b000ab88d001ddac192bce540f0e",
+  "acsdt1y2024-b25090.dat": "60f95c0745c34d9bd9d4dfca86d09947f463ef61c76d0e525b0b4b80a018810b"
+ },
+ "rows_by_level": {
+  "cd": 5232,
+  "county": 10200,
+  "national": 12,
+  "state": 612
+ },
+ "source": "ACS 2024 1-year table-based summary file (Census), https://www2.census.gov/programs-surveys/acs/summary_file/2024/table-based-SF/"
+}

experiments/us-support-mix-bakeoff-20260930/asec_quality.json

@@ -0,0 +1,159 @@
+{
+ "processed_h5_by_income_year": [
+  {
+   "income_year": 2018,
+   "survey_year": 2019,
+   "households": 68345,
+   "distinct_h_idnum": 68345,
+   "share_households_in_prior_file": null,
+   "household_weight_total_millions": 128.67448709,
+   "household_weight_cv": 0.6025358837852475,
+   "household_weight_kish_share": 0.7336490761489285,
+   "homeownership_weighted": 0.6447575960568765,
+   "unemployment_compensation_recipients": 1723,
+   "earners_whole_record_imputed_weighted": 0.242330658014494,
+   "earners_item_allocated_weighted": 0.21330270102200033
+  },
+  {
+   "income_year": 2019,
+   "survey_year": 2020,
+   "households": 60460,
+   "distinct_h_idnum": 60460,
+   "share_households_in_prior_file": 0.3777704267284155,
+   "household_weight_total_millions": 128.58819972,
+   "household_weight_cv": 0.6099413348000816,
+   "household_weight_kish_share": 0.7288478698774934,
+   "homeownership_weighted": 0.6639244541559711,
+   "unemployment_compensation_recipients": 1563,
+   "earners_whole_record_imputed_weighted": 0.2381596300674719,
+   "earners_item_allocated_weighted": 0.2125500232438193
+  },
+  {
+   "income_year": 2020,
+   "survey_year": 2021,
+   "households": 62850,
+   "distinct_h_idnum": 62850,
+   "share_households_in_prior_file": 0.32513922036595067,
+   "household_weight_total_millions": 130.0231343,
+   "household_weight_cv": 0.5979707232435548,
+   "household_weight_kish_share": 0.7366108171432274,
+   "homeownership_weighted": 0.6575969944911564,
+   "unemployment_compensation_recipients": 10915,
+   "earners_whole_record_imputed_weighted": 0.2243862480395746,
+   "earners_item_allocated_weighted": 0.22633534476314407
+  },
+  {
+   "income_year": 2021,
+   "survey_year": 2022,
+   "households": 59148,
+   "distinct_h_idnum": 59148,
+   "share_households_in_prior_file": 0.34988503415161964,
+   "household_weight_total_millions": 131.31471499,
+   "household_weight_cv": 0.607226259555364,
+   "household_weight_kish_share": 0.7306076294779332,
+   "homeownership_weighted": 0.6544510986947999,
+   "unemployment_compensation_recipients": 4065,
+   "earners_whole_record_imputed_weighted": 0.22397363113999202,
+   "earners_item_allocated_weighted": 0.21762532030886778
+  },
+  {
+   "income_year": 2022,
+   "survey_year": 2023,
+   "households": 56839,
+   "distinct_h_idnum": 56839,
+   "share_households_in_prior_file": 0.34770140220623164,
+   "household_weight_total_millions": 131.66090909,
+   "household_weight_cv": 0.599646329065834,
+   "household_weight_kish_share": 0.7355235793912631,
+   "homeownership_weighted": 0.6592966302599605,
+   "unemployment_compensation_recipients": 1225,
+   "earners_whole_record_imputed_weighted": 0.2186550960443649,
+   "earners_item_allocated_weighted": 0.2167998316454706
+  },
+  {
+   "income_year": 2023,
+   "survey_year": 2024,
+   "households": 56251,
+   "distinct_h_idnum": 56251,
+   "share_households_in_prior_file": 0.339407299425788,
+   "household_weight_total_millions": 132.39192545,
+   "household_weight_cv": 0.5979118733636303,
+   "household_weight_kish_share": 0.7366490056630862,
+   "homeownership_weighted": 0.6563332517043623,
+   "unemployment_compensation_recipients": 1360,
+   "earners_whole_record_imputed_weighted": 0.2070184806918598,
+   "earners_item_allocated_weighted": 0.20988846004956252
+  },
+  {
+   "income_year": 2024,
+   "survey_year": 2025,
+   "households": 55762,
+   "distinct_h_idnum": 55762,
+   "share_households_in_prior_file": 0.34909077866647537,
+   "household_weight_total_millions": 134.96389280000002,
+   "household_weight_cv": 0.6017557642286367,
+   "household_weight_kish_share": 0.7341550977232084,
+   "homeownership_weighted": 0.6511854704001246,
+   "unemployment_compensation_recipients": 1400,
+   "earners_whole_record_imputed_weighted": 0.1932919011043821,
+   "earners_item_allocated_weighted": 0.215243237218826
+  }
+ ],
+ "pools": {
+  "2022-2023-2024": {
+   "households": 168852,
+   "distinct_h_idnum": 130294
+  },
+  "2020-2021-2022-2023-2024": {
+   "households": 290850,
+   "distinct_h_idnum": 211834
+  }
+ },
+ "public_use_archives_by_survey_year": [
+  {
+   "survey_year": 2022,
+   "income_year": 2021,
+   "persons": 152732,
+   "whole_supplement_imputed_weighted": 0.2046639035877903,
+   "bachelors_or_higher_25plus_weighted": 0.376611355994884
+  },
+  {
+   "survey_year": 2023,
+   "income_year": 2022,
+   "persons": 146133,
+   "whole_supplement_imputed_weighted": 0.1994541476607985,
+   "bachelors_or_higher_25plus_weighted": 0.3834978999842217
+  },
+  {
+   "survey_year": 2024,
+   "income_year": 2023,
+   "persons": 144265,
+   "whole_supplement_imputed_weighted": 0.19164278967548057,
+   "bachelors_or_higher_25plus_weighted": 0.38686143531005024
+  },
+  {
+   "survey_year": 2025,
+   "income_year": 2024,
+   "persons": 142125,
+   "whole_supplement_imputed_weighted": 0.1782697407007112,
+   "bachelors_or_higher_25plus_weighted": 0.39343967585037093
+  },
+  {
+   "survey_year": 2026,
+   "income_year": 2025,
+   "persons": 134729,
+   "whole_supplement_imputed_weighted": 0.16678632658230516,
+   "bachelors_or_higher_25plus_weighted": 0.4024180140213014
+  }
+ ],
+ "combined_unweighted_response_rate_by_survey_year": {
+  "source": "CPS ASEC technical documentation, Source and Accuracy statements (combined basic-CPS x ASEC household nonresponse), transcribed 2026-09-28",
+  "2019": 0.676,
+  "2020": 0.611,
+  "2021": 0.65,
+  "2022": 0.614,
+  "2023": 0.596,
+  "2024": 0.593,
+  "2025": 0.601
+ }
+}

experiments/us-support-mix-bakeoff-20260930/asec_quality.py

@@ -0,0 +1,111 @@
+"""CPS ASEC response and processing quality by income year (receipt script).
+
+Writes ``asec_quality.json`` next to this file. Two local sources:
+
+- ``census_cps_{2018..2024}.h5`` (policyengine-us-data processed ASEC, keyed by
+  income year): households, survey year, distinct and repeated ``H_IDNUM``,
+  household-weight dispersion, homeownership, unemployment-compensation
+  recipients, and the earnings imputation flag ``I_ERNVAL`` (9 = whole record
+  imputed, 1-8 = item allocated) among wage earners.
+- the Census public-use ASEC CSV archives ``asecpub{22..26}csv.zip`` (income
+  years 2021-2025): the whole-supplement imputation flag ``FL_665`` (persons,
+  ``MARSUPWT``-weighted) and the weighted bachelor's-or-higher share among
+  adults 25 and over (``A_HGA`` >= 43).
+
+Run: ``uv run python experiments/us-support-mix-bakeoff-20260930/asec_quality.py``
+"""
+
+from __future__ import annotations
+
+import json
+import zipfile
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+STORAGE = Path("/Users/maxghenis/PolicyEngine/policyengine-us-data/policyengine_us_data/storage")
+ZIPS = Path("/Users/maxghenis/PolicyEngine/_buildm-runtime/inputs/asec_education")
+OUT = Path(__file__).resolve().parent / "asec_quality.json"
+
+
+def h5_metrics(income_year: int, previous_ids: set | None) -> tuple[dict, set]:
+    path = STORAGE / f"census_cps_{income_year}.h5"
+    with pd.HDFStore(path, "r") as store:
+        hh = store["household"][["H_IDNUM", "H_YEAR", "H_TENURE", "HSUP_WGT"]]
+        person = store["person"][["A_FNLWGT", "I_ERNVAL", "UC_VAL", "WSAL_VAL"]]
+    w = hh["HSUP_WGT"].to_numpy(np.float64) / 100
+    ids = set(hh["H_IDNUM"].astype(str))
+    earners = person[person["WSAL_VAL"] > 0]
+    ew = earners["A_FNLWGT"].to_numpy(np.float64)
+    flag = earners["I_ERNVAL"].to_numpy()
+    metrics = {
+        "income_year": income_year,
+        "survey_year": int(hh["H_YEAR"].iloc[0]),
+        "households": int(len(hh)),
+        "distinct_h_idnum": len(ids),
+        "share_households_in_prior_file": (
+            None if previous_ids is None else len(ids & previous_ids) / len(ids)
+        ),
+        "household_weight_total_millions": float(w.sum() / 1e6),
+        "household_weight_cv": float(w.std() / w.mean()),
+        "household_weight_kish_share": float(w.sum() ** 2 / np.square(w).sum() / len(w)),
+        "homeownership_weighted": float(w[hh["H_TENURE"].to_numpy() == 1].sum() / w.sum()),
+        "unemployment_compensation_recipients": int((person["UC_VAL"] > 0).sum()),
+        "earners_whole_record_imputed_weighted": float(ew[flag == 9].sum() / ew.sum()),
+        "earners_item_allocated_weighted": float(ew[(flag >= 1) & (flag <= 8)].sum() / ew.sum()),
+    }
+    return metrics, ids
+
+
+def zip_metrics(survey_year: int) -> dict:
+    path = ZIPS / f"asecpub{survey_year % 100:02d}csv.zip"
+    with zipfile.ZipFile(path) as archive:
+        name = next(n for n in archive.namelist() if n.lower().startswith("pppub") and n.lower().endswith(".csv"))
+        with archive.open(name) as handle:
+            person = pd.read_csv(handle, usecols=["FL_665", "MARSUPWT", "A_AGE", "A_HGA"])
+    w = person["MARSUPWT"].to_numpy(np.float64)
+    adults = person["A_AGE"].to_numpy() >= 25
+    return {
+        "survey_year": survey_year,
+        "income_year": survey_year - 1,
+        "persons": int(len(person)),
+        "whole_supplement_imputed_weighted": float(w[person["FL_665"].to_numpy() != 1].sum() / w.sum()),
+        "bachelors_or_higher_25plus_weighted": float(
+            w[adults & (person["A_HGA"].to_numpy() >= 43)].sum() / w[adults].sum()
+        ),
+    }
+
+
+def main() -> None:
+    files, previous = [], None
+    for year in range(2018, 2025):
+        metrics, previous = h5_metrics(year, previous)
+        files.append(metrics)
+    pools = {}
+    for years in ((2022, 2023, 2024), (2020, 2021, 2022, 2023, 2024)):
+        households, distinct = 0, set()
+        for year in years:
+            with pd.HDFStore(STORAGE / f"census_cps_{year}.h5", "r") as store:
+                ids = store["household"]["H_IDNUM"].astype(str)
+            households += len(ids)
+            distinct |= set(ids)
+        pools["-".join(map(str, years))] = {"households": households, "distinct_h_idnum": len(distinct)}
+    archives = [zip_metrics(year) for year in range(2022, 2027) if (ZIPS / f"asecpub{year % 100:02d}csv.zip").exists()]
+    payload = {
+        "processed_h5_by_income_year": files,
+        "pools": pools,
+        "public_use_archives_by_survey_year": archives,
+        "combined_unweighted_response_rate_by_survey_year": {
+            "source": "CPS ASEC technical documentation, Source and Accuracy statements "
+            "(combined basic-CPS x ASEC household nonresponse), transcribed 2026-09-28",
+            "2019": 0.676, "2020": 0.611, "2021": 0.650, "2022": 0.614,
+            "2023": 0.596, "2024": 0.593, "2025": 0.601,
+        },
+    }
+    OUT.write_text(json.dumps(payload, indent=1))
+    print(json.dumps(payload, indent=1))
+
+
+if __name__ == "__main__":
+    main()