diff --git a/changelog.d/rent-head-alignment.fixed.md b/changelog.d/rent-head-alignment.fixed.md new file mode 100644 index 0000000000..71deb73698 --- /dev/null +++ b/changelog.d/rent-head-alignment.fixed.md @@ -0,0 +1 @@ +The US ACS rent imputation now places each draw on the head of the household it was drawn for. It used to write household-order draws in person order, which left about 47% of base households with the wrong rent. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 9071607114..1d63686efd 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391", - "protocol_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f" + "map_sha256": "07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a", + "protocol_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4" }, "failures": [], "observed": { - "map_sha256": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391", - "protocol_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f" + "map_sha256": "07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a", + "protocol_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f", + "implementation_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f", + "implementation_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2" + "spec_sha256": "153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2" + "spec_sha256": "153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 2845ac1496..d8b26cb74b 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -380,8 +380,8 @@ # microcosm#1259's recipient-share bound in puf_support moves both again. # microcosm#1261 moves both again: the PUF donor carries partnership and # S-corp line-32 shares as separate leaves. - "seed_map": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391", - "seed_protocol": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f", + "seed_map": '07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a', + "seed_protocol": '4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4', "source_manifest": "608c3621b1e4a108e57c8f83d16a184acdb3d05e16a81b895f7f72dbf5d0685c", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "071d418bb49976f4682c320d2ad07097c177de3a645d2524f235e1ff33ac83f9", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py index 1f83bd82c9..f77d0a9f1b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py @@ -895,8 +895,16 @@ def impute_us_pre_subsidy_rent( if not np.isfinite(predicted).all(): raise ValueError("ACS rent QRF produced nonfinite predictions.") predicted = np.maximum(predicted, 0.0) + # ``features`` follows the household table, and the person table need not + # list heads in that order, so match each draw to its head by household id. + rent_by_household = pd.Series(predicted, index=features.index) + head_household_ids = frame.table("person")["person_household_id"].to_numpy()[ + head_mask + ] person_rent = np.zeros(frame.n("person"), dtype=np.float64) - person_rent[head_mask] = predicted + person_rent[head_mask] = rent_by_household.loc[head_household_ids].to_numpy( + dtype=np.float64 + ) return person_rent diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index de2cf00847..5837c69a8c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -18,7 +18,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # self-employment feature in the imputation schema moves the schema-set # receipt, and its post-transfer step moves acs_transfer's attested bytes. assert first.spec_sha256 == ( - "8654884dc7dde724af39b78d07cffb151c61680d388c05576cd34dee83fb79fb" + 'e8ef234504dfaed9b5cc7d04355d1996ad0a2f965c06b34679ce216c4f899abe' ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py b/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py index b75e1d8027..5221cf7803 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py @@ -1,6 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_us_housing_inputs.py.""" # ruff: noqa: F403, F405 +from hypothesis import given, settings +from hypothesis import strategies as st + from test_support.microcosm_build.us_housing_inputs import * @@ -354,3 +357,91 @@ def test_release_wiring_promotes_all_five_inputs_and_probes() -> None: ] assert contract.populace_treatment == "out_of_scope" assert contract.rate == {"status": "not_used_measured_source"} + + +def test_rent_reaches_its_own_household_when_person_order_differs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(module, "QRF", _RentQRF) + frame = _frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + tables["person"] = tables["person"].iloc[::-1].reset_index(drop=True) + reordered = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + ) + + result = with_us_housing_inputs( + reordered, seed=0, time_period=2024, acs_rent_donor=_donor() + ) + + person = result.table("person") + rent_by_household = person.set_index("person_household_id")["pre_subsidy_rent"] + tenure_by_household = result.table("household").set_index("household_id")[ + "tenure_type" + ] + renters = tenure_by_household.index[tenure_by_household == "RENTED"] + others = tenure_by_household.index[tenure_by_household != "RENTED"] + assert (rent_by_household.loc[renters] == 12_000.0).all() + assert (rent_by_household.loc[others] == 0.0).all() + + +class _OneDrawPerHouseholdFitted: + def predict(self, test: pd.DataFrame, **kwargs) -> pd.DataFrame: + return pd.DataFrame( + {"rent": 1_000.0 * np.arange(1, len(test) + 1)}, index=test.index + ) + + +class _OneDrawPerHouseholdQRF: + def __init__(self, **_kwargs: object) -> None: + pass + + def fit(self, *_args: object, **_kwargs: object) -> _OneDrawPerHouseholdFitted: + return _OneDrawPerHouseholdFitted() + + +def _frame_with_members(extra_members: list[int], *, order_seed: int) -> Frame: + frame = derive_us_housing_inputs(_frame()) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + heads = tables["person"] + members = heads.loc[np.repeat(heads.index.to_numpy(), extra_members)].copy() + members["person_id"] = heads["person_id"].max() + np.arange(1, len(members) + 1) + members["is_household_head"] = False + members["tax_unit_role_input"] = "DEPENDENT" + person = pd.concat([heads, members], ignore_index=True) + order = np.random.default_rng(order_seed).permutation(len(person)) + tables["person"] = person.iloc[order].reset_index(drop=True) + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + ) + + +@settings(max_examples=40, deadline=None) +@given( + extra_members=st.lists(st.integers(0, 3), min_size=20, max_size=20), + order_seed=st.integers(0, 2**32 - 1), +) +def test_each_head_holds_its_households_draw_under_any_person_order( + extra_members: list[int], order_seed: int +) -> None: + frame = _frame_with_members(extra_members, order_seed=order_seed) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(module, "QRF", _OneDrawPerHouseholdQRF) + rent = module.impute_us_pre_subsidy_rent(frame, _donor(), seed=0) + + person = frame.table("person") + draws = pd.Series( + 1_000.0 * np.arange(1, frame.n("household") + 1), + index=frame.table("household")["household_id"].to_numpy(), + ) + is_head = person["is_household_head"].to_numpy() + assert ( + rent[is_head] + == draws.loc[person.loc[is_head, "person_household_id"]].to_numpy() + ).all() + assert (rent[~is_head] == 0.0).all() + assert rent.sum() == draws.sum() diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 58ea9b4bbc..29915623f6 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -350,7 +350,7 @@ def capture_equality(expected: object, actual: object) -> None: # microcosm#1251's passive partnership/S-corp leaf in puf_support moves it. # microcosm#1259's recipient-share bound in puf_support moves it again. # microcosm#1261's entity split of line 32 moves it too. - "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2", + "spec_sha256": '153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0', }, }