Place each ACS rent draw on its own household's head

changelog.d/rent-head-alignment.fixed.md

@@ -0,0 +1 @@
+The US ACS rent imputation now places each draw on the head of the household it was drawn for. It used to write household-order draws in person order, which left about 47% of base households with the wrong rent.

docs/evidence/spec-engine/us-f0-coverage.json

@@ -1656,13 +1656,13 @@
           "compiler_ir.node_slices"
         ],
         "expected": {
-          "map_sha256": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391",
-          "protocol_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f"
+          "map_sha256": "07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a",
+          "protocol_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4"
         },
         "failures": [],
         "observed": {
-          "map_sha256": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391",
-          "protocol_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f"
+          "map_sha256": "07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a",
+          "protocol_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4"
         },
         "status": "covered"
       },
@@ -1677,7 +1677,7 @@
           "compiler_ir.seed_stream_map"
         ],
         "expected": {
-          "implementation_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f",
+          "implementation_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4",
           "protocol": "legacy-v1",
           "streams": [
             "build_model",
@@ -1698,7 +1698,7 @@
         },
         "failures": [],
         "observed": {
-          "implementation_sha256": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f",
+          "implementation_sha256": "4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4",
           "protocol": "legacy-v1",
           "streams": [
             "build_model",
@@ -2599,7 +2599,7 @@
       "country": "us",
       "schema_id": "country_spec",
       "schema_version": 1,
-      "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2"
+      "spec_sha256": "153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0"
     }
   },
   "report_schema_version": 3,
@@ -2609,7 +2609,7 @@
     "country": "us",
     "schema_id": "country_spec",
     "schema_version": 1,
-    "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2"
+    "spec_sha256": "153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0"
   },
   "status": "pass"
 }

packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py

@@ -380,8 +380,8 @@
     # microcosm#1259's recipient-share bound in puf_support moves both again.
     # microcosm#1261 moves both again: the PUF donor carries partnership and
     # S-corp line-32 shares as separate leaves.
-    "seed_map": "488846129a3c47e89d54538515e6bdbd7ce849e4081ab8a4ad40c9ce2a1e1391",
-    "seed_protocol": "87f44e60621c45d2fc1448fda11cd446960d6bab7cb51b346367339e2e751e1f",
+    "seed_map": '07fe9086b57a15ab1c2551fce5c30aebfa55175d76a308f7dbb268ac49da747a',
+    "seed_protocol": '4ff992324004d4c09d4165bc8d24ccff15b9d84858836851b12d532c870920f4',
     "source_manifest": "608c3621b1e4a108e57c8f83d16a184acdb3d05e16a81b895f7f72dbf5d0685c",
     "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35",
     "tail": "071d418bb49976f4682c320d2ad07097c177de3a645d2524f235e1ff33ac83f9",

packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py

@@ -895,8 +895,16 @@ def impute_us_pre_subsidy_rent(
     if not np.isfinite(predicted).all():
         raise ValueError("ACS rent QRF produced nonfinite predictions.")
     predicted = np.maximum(predicted, 0.0)
+    # ``features`` follows the household table, and the person table need not
+    # list heads in that order, so match each draw to its head by household id.
+    rent_by_household = pd.Series(predicted, index=features.index)
+    head_household_ids = frame.table("person")["person_household_id"].to_numpy()[
+        head_mask
+    ]
     person_rent = np.zeros(frame.n("person"), dtype=np.float64)
-    person_rent[head_mask] = predicted
+    person_rent[head_mask] = rent_by_household.loc[head_household_ids].to_numpy(
+        dtype=np.float64
+    )
     return person_rent
 
 

packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py

@@ -18,7 +18,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non
     # self-employment feature in the imputation schema moves the schema-set
     # receipt, and its post-transfer step moves acs_transfer's attested bytes.
     assert first.spec_sha256 == (
-        "8654884dc7dde724af39b78d07cffb151c61680d388c05576cd34dee83fb79fb"
+        'e8ef234504dfaed9b5cc7d04355d1996ad0a2f965c06b34679ce216c4f899abe'
     )
 
     second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b")

packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py

@@ -1,6 +1,9 @@
 """Tests split from packages/microcosm-build/tests/test_us_housing_inputs.py."""
 
 # ruff: noqa: F403, F405
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
 from test_support.microcosm_build.us_housing_inputs import *
 
 
@@ -354,3 +357,91 @@ def test_release_wiring_promotes_all_five_inputs_and_probes() -> None:
     ]
     assert contract.populace_treatment == "out_of_scope"
     assert contract.rate == {"status": "not_used_measured_source"}
+
+
+def test_rent_reaches_its_own_household_when_person_order_differs(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(module, "QRF", _RentQRF)
+    frame = _frame()
+    tables = {entity: frame.table(entity).copy() for entity in frame.entities}
+    tables["person"] = tables["person"].iloc[::-1].reset_index(drop=True)
+    reordered = Frame(
+        tables,
+        frame.schema,
+        {entity: frame.weights_for(entity) for entity in frame.weighted_entities},
+    )
+
+    result = with_us_housing_inputs(
+        reordered, seed=0, time_period=2024, acs_rent_donor=_donor()
+    )
+
+    person = result.table("person")
+    rent_by_household = person.set_index("person_household_id")["pre_subsidy_rent"]
+    tenure_by_household = result.table("household").set_index("household_id")[
+        "tenure_type"
+    ]
+    renters = tenure_by_household.index[tenure_by_household == "RENTED"]
+    others = tenure_by_household.index[tenure_by_household != "RENTED"]
+    assert (rent_by_household.loc[renters] == 12_000.0).all()
+    assert (rent_by_household.loc[others] == 0.0).all()
+
+
+class _OneDrawPerHouseholdFitted:
+    def predict(self, test: pd.DataFrame, **kwargs) -> pd.DataFrame:
+        return pd.DataFrame(
+            {"rent": 1_000.0 * np.arange(1, len(test) + 1)}, index=test.index
+        )
+
+
+class _OneDrawPerHouseholdQRF:
+    def __init__(self, **_kwargs: object) -> None:
+        pass
+
+    def fit(self, *_args: object, **_kwargs: object) -> _OneDrawPerHouseholdFitted:
+        return _OneDrawPerHouseholdFitted()
+
+
+def _frame_with_members(extra_members: list[int], *, order_seed: int) -> Frame:
+    frame = derive_us_housing_inputs(_frame())
+    tables = {entity: frame.table(entity).copy() for entity in frame.entities}
+    heads = tables["person"]
+    members = heads.loc[np.repeat(heads.index.to_numpy(), extra_members)].copy()
+    members["person_id"] = heads["person_id"].max() + np.arange(1, len(members) + 1)
+    members["is_household_head"] = False
+    members["tax_unit_role_input"] = "DEPENDENT"
+    person = pd.concat([heads, members], ignore_index=True)
+    order = np.random.default_rng(order_seed).permutation(len(person))
+    tables["person"] = person.iloc[order].reset_index(drop=True)
+    return Frame(
+        tables,
+        frame.schema,
+        {entity: frame.weights_for(entity) for entity in frame.weighted_entities},
+    )
+
+
+@settings(max_examples=40, deadline=None)
+@given(
+    extra_members=st.lists(st.integers(0, 3), min_size=20, max_size=20),
+    order_seed=st.integers(0, 2**32 - 1),
+)
+def test_each_head_holds_its_households_draw_under_any_person_order(
+    extra_members: list[int], order_seed: int
+) -> None:
+    frame = _frame_with_members(extra_members, order_seed=order_seed)
+    with pytest.MonkeyPatch.context() as patch:
+        patch.setattr(module, "QRF", _OneDrawPerHouseholdQRF)
+        rent = module.impute_us_pre_subsidy_rent(frame, _donor(), seed=0)
+
+    person = frame.table("person")
+    draws = pd.Series(
+        1_000.0 * np.arange(1, frame.n("household") + 1),
+        index=frame.table("household")["household_id"].to_numpy(),
+    )
+    is_head = person["is_household_head"].to_numpy()
+    assert (
+        rent[is_head]
+        == draws.loc[person.loc[is_head, "person_household_id"]].to_numpy()
+    ).all()
+    assert (rent[~is_head] == 0.0).all()
+    assert rent.sum() == draws.sum()

packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py

@@ -350,7 +350,7 @@ def capture_equality(expected: object, actual: object) -> None:
             # microcosm#1251's passive partnership/S-corp leaf in puf_support moves it.
             # microcosm#1259's recipient-share bound in puf_support moves it again.
             # microcosm#1261's entity split of line 32 moves it too.
-            "spec_sha256": "9b4b3260c20c30ba71a0947f8d78e322c8d2a3e6cb94e87a3729be950a9e90f2",
+            "spec_sha256": '153a7b6ad8f13b5de1fd32e3e4e2805015e4c90823f313cec51cb5bd27c990b0',
         },
     }