Fail the US housing gate on rent outside renter households, and zero it in the CPS draw

changelog.d/1446-rent-outside-renter-households.changed.md

@@ -0,0 +1 @@
+The US housing input gate now fails when a person outside a renter household carries positive `pre_subsidy_rent` (microcosm#1446). The support base built at c0513dbae, before the rent alignment fix in microcosm#1444, passed the gate with 66,623 such people; the base rebuilt at 9434c504b has none. `with_us_housing_inputs` returned a frame unchanged whenever the gate passed, so a base built before the fix kept its misplaced rent; it now draws that rent again, and fiscal refresh stops at the housing gate instead of releasing it. The ACS rent draw now gives heads of owner and no-cash-rent households zero rent. The shared QRF's zero-or-positive classifier gives each of them a small chance of positive rent: before this change, seeds 1 to 7 on the Route A pre-clone frame each put rent on one to four of them, and seed 0 on none.

docs/evidence/spec-engine/us-f0-coverage.json

@@ -1642,13 +1642,13 @@
           "compiler_ir.node_slices"
         ],
         "expected": {
-          "map_sha256": "2ca5e0a6c53617cc6e025086e883eab8209f153547dfdb09ea220d2ff35dfe60",
-          "protocol_sha256": "d8217ce7f319cebdbcf8125e1313ae951fc7f03bcad7ed3550f498c4cc31e269"
+          "map_sha256": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37",
+          "protocol_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53"
         },
         "failures": [],
         "observed": {
-          "map_sha256": "2ca5e0a6c53617cc6e025086e883eab8209f153547dfdb09ea220d2ff35dfe60",
-          "protocol_sha256": "d8217ce7f319cebdbcf8125e1313ae951fc7f03bcad7ed3550f498c4cc31e269"
+          "map_sha256": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37",
+          "protocol_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53"
         },
         "status": "covered"
       },
@@ -1663,7 +1663,7 @@
           "compiler_ir.seed_stream_map"
         ],
         "expected": {
-          "implementation_sha256": "d8217ce7f319cebdbcf8125e1313ae951fc7f03bcad7ed3550f498c4cc31e269",
+          "implementation_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53",
           "protocol": "legacy-v1",
           "streams": [
             "build_model",
@@ -1684,7 +1684,7 @@
         },
         "failures": [],
         "observed": {
-          "implementation_sha256": "d8217ce7f319cebdbcf8125e1313ae951fc7f03bcad7ed3550f498c4cc31e269",
+          "implementation_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53",
           "protocol": "legacy-v1",
           "streams": [
             "build_model",
@@ -2585,7 +2585,7 @@
       "country": "us",
       "schema_id": "country_spec",
       "schema_version": 1,
-      "spec_sha256": "ffa31ce070cb3d6ea508ab833a12b8fcea89faaf53a39345a54f1ad2119c62e3"
+      "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2"
     }
   },
   "report_schema_version": 3,
@@ -2595,7 +2595,7 @@
     "country": "us",
     "schema_id": "country_spec",
     "schema_version": 1,
-    "spec_sha256": "ffa31ce070cb3d6ea508ab833a12b8fcea89faaf53a39345a54f1ad2119c62e3"
+    "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2"
   },
   "status": "pass"
 }

packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py

@@ -380,8 +380,8 @@
     # microcosm#1259's recipient-share bound in puf_support moves both again.
     # microcosm#1261 moves both again: the PUF donor carries partnership and
     # S-corp line-32 shares as separate leaves.
-    "seed_map": "2ca5e0a6c53617cc6e025086e883eab8209f153547dfdb09ea220d2ff35dfe60",
-    "seed_protocol": "d8217ce7f319cebdbcf8125e1313ae951fc7f03bcad7ed3550f498c4cc31e269",
+    "seed_map": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37",
+    "seed_protocol": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53",
     "source_manifest": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6",
     "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35",
     "tail": "565f0ceeadab69d3e96a886a00dd0305f7425fa3acde24d12ba48db2b37d5fc0",

packages/microcosm-build/src/microcosm/build/us_runtime/housing_inputs.py

@@ -14,10 +14,12 @@
   and placed annual ``pre_subsidy_rent`` on the CPS household head.
 
 Microcosm preserves those source semantics and uses its shared design-weighted
-QRF.  Weighting the ACS fit is the sole deliberate strengthening of the
-retired implementation: the archived fit loaded ``household_weight`` but did
-not pass it through to microimpute.  No housing values are manufactured from
-defaults or from the retired eCPS output.
+QRF.  Weighting the ACS fit deliberately strengthens the retired
+implementation: the archived fit loaded ``household_weight`` but did not pass
+it through to microimpute.  Microcosm also sets rent to zero for heads outside
+renter households, because the shared QRF's zero-or-positive classifier gives
+each of them a small chance of positive rent.  No housing values are
+manufactured from defaults or from the retired eCPS output.
 """
 
 from __future__ import annotations
@@ -829,7 +831,12 @@ def impute_us_pre_subsidy_rent(
     seed: int,
     n_estimators: int = _DEFAULT_N_ESTIMATORS,
 ) -> np.ndarray:
-    """Draw annual ACS rent once per CPS household and place it on the head."""
+    """Draw annual ACS rent once per CPS household and place it on the head.
+
+    Heads of owner and no-cash-rent households get zero rent. The QRF's
+    zero-or-positive classifier gives each of them a small chance of a positive
+    draw, and the signal gate fails rent outside renter households.
+    """
 
     required = (
         *ACS_RENT_PREDICTORS,
@@ -875,6 +882,7 @@ def impute_us_pre_subsidy_rent(
 
         QRF = import_module("microcosm.fit").QRF
     features, head_mask = _recipient_head_features(frame)
+    renter = features["tenure_type"].astype(str).eq("RENTED").to_numpy()
     encoded_training, encoded_features, encoded_predictors = _encode_acs_predictors(
         fit_frame,
         features,
@@ -895,6 +903,8 @@ def impute_us_pre_subsidy_rent(
     if not np.isfinite(predicted).all():
         raise ValueError("ACS rent QRF produced nonfinite predictions.")
     predicted = np.maximum(predicted, 0.0)
+    # Every head draws, so zeroing afterwards leaves renters' draws unchanged.
+    predicted = np.where(renter, predicted, 0.0)
     # ``features`` follows the household table, and the person table need not
     # list heads in that order, so match each draw to its head by household id.
     rent_by_household = pd.Series(predicted, index=features.index)
@@ -1162,7 +1172,13 @@ def with_us_housing_inputs(
     acs_rent_donor: pd.DataFrame,
     n_estimators: int = _DEFAULT_N_ESTIMATORS,
 ) -> Frame:
-    """Materialize source-backed housing and tenure inputs on a US frame."""
+    """Materialize source-backed housing and tenure inputs on a US frame.
+
+    A frame that already passes the signal gate returns unchanged. Any other
+    frame gets all five inputs again: ``H_TENURE``, ``SPM_CAPHOUSESUB`` and
+    ``SPM_TENMORTSTATUS`` supply tenure and assistance, and the ACS rent model
+    draws ``pre_subsidy_rent`` for renter heads. Callers gate the result.
+    """
 
     if frame.schema != US_SCHEMA:
         raise ValueError("US housing inputs require the US schema.")
@@ -1289,7 +1305,12 @@ def us_housing_inputs_summary(frame: Frame) -> dict[str, object]:
 
 
 def us_housing_inputs_signal_gate(frame: Frame) -> GateResult:
-    """Require plausible non-default signal for all five restored inputs."""
+    """Require plausible non-default signal for all five restored inputs.
+
+    Positive rent may sit only on the head of a renter household. Rent written
+    to anyone in an owner or no-cash-rent household fails; rent moved from one
+    renter household's head to another's passes.
+    """
 
     missing: list[str] = []
     for entity, columns in (
@@ -1354,6 +1375,7 @@ def us_housing_inputs_signal_gate(frame: Frame) -> GateResult:
         ("nonfinite_rent", "nonfinite pre_subsidy_rent"),
         ("negative_rent", "negative pre_subsidy_rent"),
         ("positive_rent_nonhead", "positive rent on non-head people"),
+        ("positive_rent_nonrenter", "positive rent outside renter households"),
     ):
         count = int(summary[key])
         if count:

packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py

@@ -25,7 +25,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non
     # ssi_disability_criteria's attested bytes.
     # The opt-in cross-fitted gate calibration moves the fit kernels' bytes.
     assert first.spec_sha256 == (
-        "800eb80fb737f304df33c3f856ea512fa7f8a4bc45a8c456e60a9514c1026e69"
+        "5b39d83cafb6f312ce73e5550ab36da6676ce265779d615f49c7987f5cc5d36e"
     )
 
     second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b")

packages/microcosm-build/tests/engine_free/us/test_us_housing_inputs.py

@@ -164,7 +164,7 @@ def fit(
     assert captured["weights"] == "household_weight"
 
 
-def test_signal_gate_preserves_archived_qrf_rent_outside_renter_household(
+def test_signal_gate_fails_rent_outside_renter_household(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
     monkeypatch.setattr(module, "QRF", _RentQRF)
@@ -175,10 +175,75 @@ def test_signal_gate_preserves_archived_qrf_rent_outside_renter_household(
 
     gate = us_housing_inputs_signal_gate(result)
 
-    assert gate.passed, gate.failures
+    assert not gate.passed
+    assert gate.failures == ("positive rent outside renter households: 1.",)
     assert gate.details["positive_rent_nonrenter"] == 1
 
 
+def _with_rent_in_person_order(frame: Frame) -> tuple[Frame, int]:
+    """Write one draw per household to heads in person order, as before #1444.
+
+    Returns the frame and the number of heads outside renter households that
+    end up holding a renter's draw.
+    """
+
+    household = frame.table("household")
+    tenure = household["tenure_type"].to_numpy()
+    draws = np.where(tenure == "RENTED", 12_000.0, 0.0)
+    tables = {entity: frame.table(entity).copy() for entity in frame.entities}
+    person = tables["person"]
+    head = person["is_household_head"].to_numpy()
+    rent = np.zeros(len(person), dtype=np.float64)
+    rent[head] = draws
+    person["pre_subsidy_rent"] = rent
+    head_tenure = (
+        person.loc[head, "person_household_id"]
+        .map(pd.Series(tenure, index=household["household_id"].to_numpy()))
+        .to_numpy()
+    )
+    moved = int(np.count_nonzero((draws > 0.0) & (head_tenure != "RENTED")))
+    return (
+        Frame(
+            tables,
+            frame.schema,
+            {entity: frame.weights_for(entity) for entity in frame.weighted_entities},
+        ),
+        moved,
+    )
+
+
+def test_signal_gate_fails_rent_written_in_person_order_and_redraws_it(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(module, "QRF", _RentQRF)
+    frame = derive_us_housing_inputs(_frame())
+    tables = {entity: frame.table(entity).copy() for entity in frame.entities}
+    tables["person"] = tables["person"].iloc[::-1].reset_index(drop=True)
+    reordered = Frame(
+        tables,
+        frame.schema,
+        {entity: frame.weights_for(entity) for entity in frame.weighted_entities},
+    )
+    misplaced, moved = _with_rent_in_person_order(reordered)
+
+    gate = us_housing_inputs_signal_gate(misplaced)
+
+    # The four renter draws land on the heads of households 20 to 17, all owners.
+    assert moved == 4
+    assert not gate.passed
+    assert gate.failures == ("positive rent outside renter households: 4.",)
+
+    redrawn = with_us_housing_inputs(
+        misplaced, seed=0, time_period=2024, acs_rent_donor=_donor()
+    )
+
+    assert us_housing_inputs_signal_gate(redrawn).passed
+    person = redrawn.table("person")
+    rent = person.set_index("person_household_id")["pre_subsidy_rent"]
+    assert (rent.loc[[1, 2, 3, 4]] == 12_000.0).all()
+    assert (rent.drop([1, 2, 3, 4]) == 0.0).all()
+
+
 def test_puf_half_reimputes_only_housing_assistance(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -402,7 +467,28 @@ def fit(self, *_args: object, **_kwargs: object) -> _OneDrawPerHouseholdFitted:
         return _OneDrawPerHouseholdFitted()
 
 
-def _frame_with_members(extra_members: list[int], *, order_seed: int) -> Frame:
+def test_rent_drawn_for_owner_and_no_cash_rent_heads_is_zeroed(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # The stub draws positive rent for every household, as the real QRF can for
+    # owners: its zero-or-positive classifier never gives them probability 0.
+    monkeypatch.setattr(module, "QRF", _OneDrawPerHouseholdQRF)
+
+    result = with_us_housing_inputs(
+        _frame(), seed=0, time_period=2024, acs_rent_donor=_donor()
+    )
+
+    rent = result.table("person").set_index("person_household_id")["pre_subsidy_rent"]
+    assert rent.loc[[1, 2, 3, 4]].tolist() == [1_000.0, 2_000.0, 3_000.0, 4_000.0]
+    assert (rent.drop([1, 2, 3, 4]) == 0.0).all()
+    gate = us_housing_inputs_signal_gate(result)
+    assert gate.details["positive_rent_nonrenter"] == 0
+    assert gate.passed, gate.failures
+
+
+def _frame_with_members(
+    extra_members: list[int], *, order_seed: int, shuffle: bool = True
+) -> Frame:
     frame = derive_us_housing_inputs(_frame())
     tables = {entity: frame.table(entity).copy() for entity in frame.entities}
     heads = tables["person"]
@@ -411,7 +497,12 @@ def _frame_with_members(extra_members: list[int], *, order_seed: int) -> Frame:
     members["is_household_head"] = False
     members["tax_unit_role_input"] = "DEPENDENT"
     person = pd.concat([heads, members], ignore_index=True)
-    order = np.random.default_rng(order_seed).permutation(len(person))
+    # Unshuffled, heads come first in household order.
+    order = (
+        np.random.default_rng(order_seed).permutation(len(person))
+        if shuffle
+        else np.arange(len(person))
+    )
     tables["person"] = person.iloc[order].reset_index(drop=True)
     return Frame(
         tables,
@@ -425,7 +516,7 @@ def _frame_with_members(extra_members: list[int], *, order_seed: int) -> Frame:
     extra_members=st.lists(st.integers(0, 3), min_size=20, max_size=20),
     order_seed=st.integers(0, 2**32 - 1),
 )
-def test_each_head_holds_its_households_draw_under_any_person_order(
+def test_each_renter_head_holds_its_households_draw_under_any_person_order(
     extra_members: list[int], order_seed: int
 ) -> None:
     frame = _frame_with_members(extra_members, order_seed=order_seed)
@@ -434,9 +525,11 @@ def test_each_head_holds_its_households_draw_under_any_person_order(
         rent = module.impute_us_pre_subsidy_rent(frame, _donor(), seed=0)
 
     person = frame.table("person")
+    household = frame.table("household")
+    renter = household["tenure_type"].eq("RENTED").to_numpy()
     draws = pd.Series(
-        1_000.0 * np.arange(1, frame.n("household") + 1),
-        index=frame.table("household")["household_id"].to_numpy(),
+        np.where(renter, 1_000.0 * np.arange(1, frame.n("household") + 1), 0.0),
+        index=household["household_id"].to_numpy(),
     )
     is_head = person["is_household_head"].to_numpy()
     assert (
@@ -445,3 +538,76 @@ def test_each_head_holds_its_households_draw_under_any_person_order(
     ).all()
     assert (rent[~is_head] == 0.0).all()
     assert rent.sum() == draws.sum()
+
+
+@settings(max_examples=40, deadline=None)
+@given(
+    extra_members=st.lists(st.integers(0, 3), min_size=20, max_size=20),
+    order_seed=st.integers(0, 2**32 - 1),
+    shuffle=st.booleans(),
+)
+def test_gate_flags_exactly_the_draws_person_order_moves_off_renters(
+    extra_members: list[int], order_seed: int, shuffle: bool
+) -> None:
+    frame = _frame_with_members(extra_members, order_seed=order_seed, shuffle=shuffle)
+    misplaced, moved = _with_rent_in_person_order(frame)
+    if not shuffle:
+        assert moved == 0
+
+    gate = us_housing_inputs_signal_gate(misplaced)
+
+    assert gate.details["positive_rent_nonrenter"] == moved
+    flagged = [f for f in gate.failures if "outside renter households" in f]
+    expected = [f"positive rent outside renter households: {moved}."] if moved else []
+    assert flagged == expected
+
+
+@settings(max_examples=40, deadline=None)
+@given(
+    extra_members=st.lists(st.integers(0, 3), min_size=20, max_size=20),
+    order_seed=st.integers(0, 2**32 - 1),
+)
+def test_drawn_rent_passes_the_renter_check_when_the_model_gives_owners_rent(
+    extra_members: list[int], order_seed: int
+) -> None:
+    frame = _frame_with_members(extra_members, order_seed=order_seed)
+    with pytest.MonkeyPatch.context() as patch:
+        patch.setattr(module, "QRF", _OneDrawPerHouseholdQRF)
+        result = with_us_housing_inputs(
+            frame, seed=0, time_period=2024, acs_rent_donor=_donor()
+        )
+
+    gate = us_housing_inputs_signal_gate(result)
+
+    assert gate.details["positive_rent_nonrenter"] == 0
+    assert gate.passed, gate.failures
+
+
+@settings(max_examples=60, deadline=None)
+@given(rent=st.lists(st.sampled_from([0.0, 1.0, 12_000.0]), min_size=20, max_size=20))
+def test_gate_counts_every_positive_rent_outside_renter_households(
+    rent: list[float],
+) -> None:
+    with pytest.MonkeyPatch.context() as patch:
+        patch.setattr(module, "QRF", _RentQRF)
+        result = with_us_housing_inputs(
+            _frame(), seed=0, time_period=2024, acs_rent_donor=_donor()
+        )
+    person = result.table("person")
+    person["pre_subsidy_rent"] = rent
+    household = result.table("household")
+    renters = set(household.loc[household["tenure_type"] == "RENTED", "household_id"])
+    expected = sum(
+        1
+        for value, household_id in zip(rent, person["person_household_id"], strict=True)
+        if value > 0.0 and household_id not in renters
+    )
+
+    gate = us_housing_inputs_signal_gate(result)
+
+    assert gate.details["positive_rent_nonrenter"] == expected
+    flagged = [f for f in gate.failures if "outside renter households" in f]
+    expected_failures = (
+        [f"positive rent outside renter households: {expected}."] if expected else []
+    )
+    assert flagged == expected_failures

packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py

@@ -353,7 +353,7 @@ def capture_equality(expected: object, actual: object) -> None:
             # The opt-in cross-fitted gate calibration in microcosm.fit.qrf moves it.
             # microcosm#1073's scf_wealth education-debt declaration moves it
             # through the bundle's source stages.
-            "spec_sha256": "ffa31ce070cb3d6ea508ab833a12b8fcea89faaf53a39345a54f1ad2119c62e3",
+            "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2",
         },
     }