From 628c8248a5e5dfcb305f487737661882cc573c9e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 18:03:29 -0500 Subject: [PATCH 1/3] Place each ACS rent draw on its own household's head impute_us_pre_subsidy_rent drew rent in household-table order and wrote it through a head mask in person-table order. The two orders differ on every US base, so 91% of base households carry another household's rent, and rent lands almost independently of tenure. Match draws to heads by household id instead, and pin the behaviour with a test that reverses the person table. Co-Authored-By: Claude Opus 5.5 --- changelog.d/rent-head-alignment.fixed.md | 1 + .../build/us_runtime/housing_inputs.py | 10 ++++++- .../engine_free/us/test_us_housing_inputs.py | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 changelog.d/rent-head-alignment.fixed.md diff --git a/changelog.d/rent-head-alignment.fixed.md b/changelog.d/rent-head-alignment.fixed.md new file mode 100644 index 0000000000..a838e4ba57 --- /dev/null +++ b/changelog.d/rent-head-alignment.fixed.md @@ -0,0 +1 @@ +The US ACS rent imputation now places each draw on the head of the household it was drawn for. It used to write household-order draws in person order, which left about 91% of base households with another household's rent. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py index 1f83bd82c9..f77d0a9f1b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py @@ -895,8 +895,16 @@ def impute_us_pre_subsidy_rent( if not np.isfinite(predicted).all(): raise ValueError("ACS rent QRF produced nonfinite predictions.") predicted = np.maximum(predicted, 0.0) + # ``features`` follows the household table, and the person table need not + # list heads in that order, so match each draw to its head by household id. + rent_by_household = pd.Series(predicted, index=features.index) + head_household_ids = frame.table("person")["person_household_id"].to_numpy()[ + head_mask + ] person_rent = np.zeros(frame.n("person"), dtype=np.float64) - person_rent[head_mask] = predicted + person_rent[head_mask] = rent_by_household.loc[head_household_ids].to_numpy( + dtype=np.float64 + ) return person_rent diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py b/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py index b75e1d8027..41cd0ffc6b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py @@ -354,3 +354,31 @@ def test_release_wiring_promotes_all_five_inputs_and_probes() -> None: ] assert contract.populace_treatment == "out_of_scope" assert contract.rate == {"status": "not_used_measured_source"} + + +def test_rent_reaches_its_own_household_when_person_order_differs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(module, "QRF", _RentQRF) + frame = _frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = tables["person"].iloc[::-1].reset_index(drop=True) + reordered = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + ) + + result = with_us_housing_inputs( + reordered, seed=0, time_period=2024, acs_rent_donor=_donor() + ) + + person = result.table("person") + rent_by_household = person.set_index("person_household_id")["pre_subsidy_rent"] + tenure_by_household = result.table("household").set_index("household_id")[ + "tenure_type" + ] + renters = tenure_by_household.index[tenure_by_household == "RENTED"] + others = tenure_by_household.index[tenure_by_household != "RENTED"] + assert (rent_by_household.loc[renters] == 12_000.0).all() + assert (rent_by_household.loc[others] == 0.0).all() From 62c96f55d26150a1b13cf900273847b73b09675b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 18:54:25 -0500 Subject: [PATCH 2/3] Repin the attested housing kernel and test rent under any person order housing_inputs.py is an attested US kernel, so the rent fix moves its seed map, seed protocol and spec digests. This commit applies the documented repin sequence and regenerates the coverage receipt. The new property test gives each household a distinct draw, adds multi-person households and shuffles the person table. It fails on the old assignment. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++--- .../build/spec_engine/inventory_coverage.py | 4 +- .../shared/test_spec_engine_loader.py | 2 +- .../engine_free/us/test_us_housing_inputs.py | 63 +++++++++++++++++++ .../us/test_us_multispine_pool_tool.py | 2 +- 5 files changed, 75 insertions(+), 12 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 9071607114..1d63686efd 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391", - "protocol_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f" + "map_sha256": "07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a", + "protocol_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4" }, "failures": [], "observed": { - "map_sha256": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391", - "protocol_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f" + "map_sha256": "07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a", + "protocol_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f", + "implementation_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f", + "implementation_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2" + "spec_sha256": "153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2" + "spec_sha256": "153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 2845ac1496..d8b26cb74b 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -380,8 +380,8 @@ # microcosm#1259's recipient-share bound in puf_support moves both again. # microcosm#1261 moves both again: the PUF donor carries partnership and # S-corp line-32 shares as separate leaves. - "seed_map": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391", - "seed_protocol": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f", + "seed_map": '07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a', + "seed_protocol": '4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4', "source_manifest": "608c3621b1e4a108e57c8f83d16a184acdb3d05e16a81b895f7f72dbf5d0685c", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "071d418bb49976f4682c320d2ad07097c177de3a645d2524f235e1ff33ac83f9", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index de2cf00847..5837c69a8c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -18,7 +18,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # self-employment feature in the imputation schema moves the schema-set # receipt, and its post-transfer step moves acs_transfer's attested bytes. assert first.spec_sha256 == ( - "8654884dc7dde724af39b78d07cffb151c61680d388c05576cd34dee83fb79fb" + 'e8ef234504dfaed9b5cc7d04355d1996ad0a2f965c06b34679ce216c4f899abe' ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py b/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py index 41cd0ffc6b..5221cf7803 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py @@ -1,6 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_us_housing_inputs.py.""" # ruff: noqa: F403, F405 +from hypothesis import given, settings +from hypothesis import strategies as st + from test_support.microcosm_build.us_housing_inputs import * @@ -382,3 +385,63 @@ def test_rent_reaches_its_own_household_when_person_order_differs( others = tenure_by_household.index[tenure_by_household != "RENTED"] assert (rent_by_household.loc[renters] == 12_000.0).all() assert (rent_by_household.loc[others] == 0.0).all() + + +class _OneDrawPerHouseholdFitted: + def predict(self, test: pd.DataFrame, **kwargs) -> pd.DataFrame: + return pd.DataFrame( + {"rent": 1_000.0 * np.arange(1, len(test) + 1)}, index=test.index + ) + + +class _OneDrawPerHouseholdQRF: + def __init__(self, **_kwargs: object) -> None: + pass + + def fit(self, *_args: object, **_kwargs: object) -> _OneDrawPerHouseholdFitted: + return _OneDrawPerHouseholdFitted() + + +def _frame_with_members(extra_members: list[int], *, order_seed: int) -> Frame: + frame = derive_us_housing_inputs(_frame()) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + heads = tables["person"] + members = heads.loc[np.repeat(heads.index.to_numpy(), extra_members)].copy() + members["person_id"] = heads["person_id"].max() + np.arange(1, len(members) + 1) + members["is_household_head"] = False + members["tax_unit_role_input"] = "DEPENDENT" + person = pd.concat([heads, members], ignore_index=True) + order = np.random.default_rng(order_seed).permutation(len(person)) + tables["person"] = person.iloc[order].reset_index(drop=True) + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + ) + + +@settings(max_examples=40, deadline=None) +@given( + extra_members=st.lists(st.integers(0, 3), min_size=20, max_size=20), + order_seed=st.integers(0, 2**32 - 1), +) +def test_each_head_holds_its_households_draw_under_any_person_order( + extra_members: list[int], order_seed: int +) -> None: + frame = _frame_with_members(extra_members, order_seed=order_seed) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(module, "QRF", _OneDrawPerHouseholdQRF) + rent = module.impute_us_pre_subsidy_rent(frame, _donor(), seed=0) + + person = frame.table("person") + draws = pd.Series( + 1_000.0 * np.arange(1, frame.n("household") + 1), + index=frame.table("household")["household_id"].to_numpy(), + ) + is_head = person["is_household_head"].to_numpy() + assert ( + rent[is_head] + == draws.loc[person.loc[is_head, "person_household_id"]].to_numpy() + ).all() + assert (rent[~is_head] == 0.0).all() + assert rent.sum() == draws.sum() diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 58ea9b4bbc..29915623f6 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -350,7 +350,7 @@ def capture_equality(expected: object, actual: object) -> None: # microcosm#1251's passive partnership/S-corp leaf in puf_support moves it. # microcosm#1259's recipient-share bound in puf_support moves it again. # microcosm#1261's entity split of line 32 moves it too. - "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2", + "spec_sha256": '153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0', }, } From 233f459c4aeda603fe15f1592c0295eb112bcf64 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 19:50:48 -0500 Subject: [PATCH 3/3] Correct the rent fix changelog figure About 91% of households hold another household's draw, but many of those draws are zero both ways. About 47% hold the wrong rent. Co-Authored-By: Claude Opus 5.5 --- changelog.d/rent-head-alignment.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/rent-head-alignment.fixed.md b/changelog.d/rent-head-alignment.fixed.md index a838e4ba57..71deb73698 100644 --- a/changelog.d/rent-head-alignment.fixed.md +++ b/changelog.d/rent-head-alignment.fixed.md @@ -1 +1 @@ -The US ACS rent imputation now places each draw on the head of the household it was drawn for. It used to write household-order draws in person order, which left about 91% of base households with another household's rent. +The US ACS rent imputation now places each draw on the head of the household it was drawn for. It used to write household-order draws in person order, which left about 47% of base households with the wrong rent.