From 0cabbc9995ae4b724294ea00ba9a86641f475b33 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Mon, 12 Oct 2026 13:47:21 -0400 Subject: [PATCH 01/17] Impute federal student loan balances from the 2025 SCF (#1073) Add the scf_education_debt runtime module. It carries federal education debt from the 2025 SCF through two weighted QRFs and allocates it to the SCF-style reference person and spouse as four person inputs: federal_student_loan_balance and federal_parent_plus_loan_balance (policyengine-us#9721), is_income_based_repayment_new_borrower and federal_student_loan_graduate_share (policyengine-us#9724). - Only loans flagged federal count; X7179 is spread over loans 1-6 in proportion to their balances; whose-education code 3 is Parent PLUS on the reference person, codes 6, 7 and -7 the reference person's own. - scf_wealth declares the two pinned 2025 files as artifacts 3 and 4 with a new scf_2025 vintage. Net worth, liquid assets and auto loans stay on 2022. - The fiscal-refresh tool runs the stage only with --scf-education-debt. The four columns join the stored-input register with the pending- engine reason the parent ids carry; the coverage manifest is left until the engine pin defines them. - Regenerate sources, vintages and calibration YAML and re-pin the frozen source_stages.json and calibration projection digests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-2025-education-debt.added.md | 11 + .../build_1073_scf_education_debt_receipt.py | 734 ++++++++ .../src/microcosm/build/us/source_stages.json | 180 +- .../microcosm/build/us/spec/calibration.yaml | 4 + .../src/microcosm/build/us/spec/sources.yaml | 171 +- .../src/microcosm/build/us/spec/vintages.yaml | 10 + .../microcosm/build/us_runtime/__init__.py | 41 +- .../build/us_runtime/scf_education_debt.py | 1611 +++++++++++++++++ .../test_spec_engine_calibration_semantics.py | 2 +- .../shared/test_spec_engine_compiler_ir.py | 2 +- .../us/test_us_bundle_core_contracts.py | 7 +- .../us/test_us_fiscal_refresh_builder.py | 102 ++ .../us/test_us_scf_education_debt.py | 671 +++++++ .../engine_free/us/test_us_spine_blindness.py | 3 + .../us/test_us_stored_input_register.py | 10 +- .../src/microcosm/data/stored_inputs.py | 41 +- .../microcosm_build/us_spec_bundle.py | 2 +- .../us_stored_input_register.py | 3 + tools/build_us_fiscal_refresh_release.py | 123 ++ tools/generate_us_bundle_from_constants.py | 2 +- tools/us_bundle_generation/core.py | 12 + 21 files changed, 3723 insertions(+), 19 deletions(-) create mode 100644 changelog.d/1073-scf-2025-education-debt.added.md create mode 100644 experiments/build_1073_scf_education_debt_receipt.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py diff --git a/changelog.d/1073-scf-2025-education-debt.added.md b/changelog.d/1073-scf-2025-education-debt.added.md new file mode 100644 index 0000000000..493506b44d --- /dev/null +++ b/changelog.d/1073-scf-2025-education-debt.added.md @@ -0,0 +1,11 @@ +The US build can impute federal student loan balances from the 2025 Survey of Consumer Finances (microcosm#1073, the EqRC student debt pilot). A new runtime module, `scf_education_debt`, writes four person inputs: `federal_student_loan_balance` and `federal_parent_plus_loan_balance` (policyengine-us#9721), and `is_income_based_repayment_new_borrower` and `federal_student_loan_graduate_share` (policyengine-us#9724). + +Only education debt reads the 2025 SCF. Net worth, the liquid-asset leaves and auto loans stay on the 2022 SCF. The `scf_wealth` stage declares the two 2025 files (`scfp2025s.zip`, `scf2025s.zip`), pinned by zip and member SHA-256 against a fresh Federal Reserve download, as artifacts 3 and 4, and a `scf_2025` vintage points at them. Moving the whole stage to 2025 later changes pinned coordinates, not code: the 2025 summary extract carries every column the shared predictor construction reads. + +The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`; loans for a child go to the reference person's `federal_parent_plus_loan_balance`; codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. A person is an IBR new borrower when every federal loan they hold was borrowed in 2014 or later, and the graduate share weights each own loan's program by the amount borrowed. + +The model. One weighted QRF draws each household's federal education debt on the eight shared SCF predictors plus household income less capital gains. A second, fit on borrower families and conditioning on that total, draws the holder shares, graduate shares and new-borrower flags. The SCF-style reference person and the member its `A_SPOUSE` (else `PECOHAB`) pointer names receive the balances. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. + +The fiscal-refresh release tool runs the stage only with `--scf-education-debt`, because the pinned policyengine-us does not define the four inputs yet. The four columns join `US_STORED_NON_VARIABLE_COLUMNS` with the pending-engine reason the parent ids carry, and leave it when the engine pin moves. They are not in the release input-coverage manifest, whose check requires every column to be an engine input. No release is certified with them. + +Fit receipts are in `experiments/1073-scf-education-debt-receipts.md`, generated by `experiments/build_1073_scf_education_debt_receipt.py`. diff --git a/experiments/build_1073_scf_education_debt_receipt.py b/experiments/build_1073_scf_education_debt_receipt.py new file mode 100644 index 0000000000..7c8068b005 --- /dev/null +++ b/experiments/build_1073_scf_education_debt_receipt.py @@ -0,0 +1,734 @@ +#!/usr/bin/env python +"""Fit receipts for the SCF 2025 federal student loan stage (microcosm#1073). + +Three measurements, all weighted, all against the 2025 SCF: + +1. **Source checks.** The loader's education debt against the Fed's published + Alternate Table 13 (19.92% of families hold education debt, median + $25,500), and how the federal flag, X7179 and the whose-education codes + split the balances the X7179 and holder rules act on. +2. **Hold-out fit and predictor ablation.** Families are split into five folds + by ``yy1`` (all implicates of a family stay together). Each fold is drawn + from the stage's two QRFs fit on the other four, and the drawn table is + compared with the observed one, for each predictor set. +3. **Host fit.** The stage imputed onto a US Microcosm H5 (the published + default unless ``--h5`` names another), tabulated at that file's weights. + +Nothing here is a gate or a calibration target. Run with the repository's +engine-free environment: + + .venv/Scripts/python.exe experiments/build_1073_scf_education_debt_receipt.py \ + --scf-zip-dir ~/.cache/scf --h5 populace_us_2024.h5 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_BALANCE_BANDS, + SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + ScfPublicFile, + _read_stata_columns, + draw_scf_education_debt_households, + fetch_scf_public_file, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, + with_us_scf_education_debt_inputs, +) +from microcosm.build.us_runtime.scf_wealth import SCF_WEALTH_PREDICTORS +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +ISSUE = "PolicyEngine/microcosm#1073" +FED_ALT_TABLE_13 = { + "source": ( + "Federal Reserve, 2025 SCF Chartbook tables, scf2025_tables_public_real_" + "historical.xlsx, sheet 'Table 13 25 Alt %s & median', education loans" + ), + "share_of_families_with_education_debt_pct": 19.92, + "median_among_holders": 25_500.0, +} +#: The published default the host fit imputes onto, and its release-manifest +#: SHA-256 (releases/populace-us-2024-spm-20260915/release_manifest.json). +DEFAULT_HOST_RELEASE = "populace-us-2024-spm-20260915" +DEFAULT_HOST_SHA256 = "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" +PRODUCTION_VARIANT = "shared_8_plus_household_income" +VARIANTS: Mapping[str, tuple[str, ...]] = { + "shared_8_without_age": tuple(p for p in SCF_WEALTH_PREDICTORS if p != "age"), + "shared_8": SCF_WEALTH_PREDICTORS, + PRODUCTION_VARIANT: SCF_EDUCATION_DEBT_PREDICTORS, + "plus_household_income_and_education": (*SCF_EDUCATION_DEBT_PREDICTORS, "educ"), +} +WHOSE_LABELS = { + 1: "1 respondent (reference person)", + 2: "2 spouse or partner", + 3: "3 child (incl. grandchild, other relative)", + 6: "6 multiple people", + 7: "7 unrelated person", + -7: "-7 other", +} +_HOST_PERSON_COLUMNS = ( + "person_id", + "person_household_id", + "person_tax_unit_id", + "person_spm_unit_id", + "person_family_id", + "person_marital_unit_id", + "age", + "is_female", + "PRDTRACE", + "PRDTHSP", + "A_MARITL", + "A_LINENO", + "A_SPOUSE", + "PECOHAB", + "PH_SEQ", + "PEPAR1", + "PEPAR2", +) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _member_from_local_zip( + public_file: ScfPublicFile, zip_dir: Path, cache_dir: Path +) -> Path: + """Extract a pinned member from a locally cached Fed zip, verifying both pins.""" + + archive = zip_dir / public_file.zip_name + observed = _sha256(archive) + if observed != public_file.zip_sha256: + raise ValueError(f"{archive} sha-256 {observed} != {public_file.zip_sha256}") + cache_dir.mkdir(parents=True, exist_ok=True) + target = cache_dir / public_file.member + with zipfile.ZipFile(archive) as handle: + target.write_bytes(handle.read(public_file.member)) + observed = _sha256(target) + if observed != public_file.member_sha256: + raise ValueError(f"{target} sha-256 {observed} != {public_file.member_sha256}") + return target + + +def _pooled_median(values: np.ndarray, weights: np.ndarray) -> float: + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, cumulative[-1] / 2, side="left")) + return float(values[order][index]) + + +def source_checks(donor: pd.DataFrame, full: pd.DataFrame) -> dict[str, object]: + """The loader against the Fed's table, and the balances each rule acts on.""" + + weight = donor["scf_weight"].to_numpy(dtype=np.float64) + debt = donor["education_debt"].to_numpy(dtype=np.float64) + holders = debt > 0 + per_implicate = [ + 100.0 + * float(group.loc[group["education_debt"] > 0, "scf_weight"].sum()) + / float(group["scf_weight"].sum()) + for _, group in donor.groupby("implicate") + ] + full = full.reset_index(drop=True) + full_weight = full["wgt"].to_numpy(dtype=np.float64) + balances = np.column_stack( + [ + np.maximum(full[c].to_numpy(dtype=np.float64), 0.0) + for c in SCF_EDUCATION_LOAN_BALANCE_COLUMNS + ] + ) + federal = np.column_stack( + [full[c].to_numpy(dtype=np.float64) for c in SCF_EDUCATION_LOAN_FEDERAL_COLUMNS] + ) + whose = np.column_stack( + [full[c].to_numpy(dtype=np.float64) for c in SCF_EDUCATION_LOAN_WHOSE_COLUMNS] + ) + remaining = np.maximum( + full[SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN].to_numpy(dtype=np.float64), + 0.0, + ) + dollars = balances * full_weight[:, None] + loans_1_6 = float(dollars.sum()) + all_education = loans_1_6 + float(np.dot(remaining, full_weight)) + federal_dollars = np.where(federal == 1, dollars, 0.0) + by_whose = {} + for code, label in WHOSE_LABELS.items(): + by_whose[label] = float(federal_dollars[whose == code].sum()) / float( + federal_dollars.sum() + ) + other_codes = sorted( + {int(code) for code in np.unique(whose[balances > 0])} - set(WHOSE_LABELS) + ) + return { + "fed_alternate_table_13": FED_ALT_TABLE_13, + "loader_share_of_families_with_education_debt_pct": float( + np.mean(per_implicate) + ), + "loader_median_among_holders": _pooled_median(debt[holders], weight[holders]), + "families_per_implicate": int(len(donor) // 5), + "balance_shares": { + "loans_1_6_of_all_education_debt": loans_1_6 / all_education, + "x7179_remaining_loans_of_all_education_debt": 1.0 + - loans_1_6 / all_education, + "federal_of_loans_1_6": float(federal_dollars.sum()) / loans_1_6, + "non_federal_of_loans_1_6": float(np.where(federal == 5, dollars, 0).sum()) + / loans_1_6, + "federal_status_blank_of_loans_1_6": float( + np.where(federal == 0, dollars, 0).sum() + ) + / loans_1_6, + }, + "federal_balance_share_by_whose_education_code": by_whose, + "whose_education_codes_outside_the_rule": other_codes, + "federal_education_debt_after_x7179_rule": float( + np.dot(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], weight) + ), + "remaining_loans_without_detail_dollars": float( + np.dot(donor["remaining_loans_without_detail"], weight) + ), + } + + +def _fold_of_family(families: np.ndarray, *, folds: int, seed: int) -> np.ndarray: + unique = np.unique(families) + rng = np.random.default_rng(np.random.SeedSequence([seed, 1073, 5])) + assignment = pd.Series(rng.permutation(len(unique)) % folds, index=unique) + return assignment.reindex(families).to_numpy() + + +def holdout_draws( + donor: pd.DataFrame, + predictors: Sequence[str], + *, + folds: int, + seed: int, + n_estimators: int, +) -> pd.DataFrame: + """Draw every SCF row from models fit on the other folds' families.""" + + fold = _fold_of_family(donor["yy1"].to_numpy(), folds=folds, seed=seed) + parts = [] + for k in range(folds): + held_out = fold == k + drawn = draw_scf_education_debt_households( + donor.loc[held_out, list(predictors)], + donor.loc[~held_out], + seed=seed + k, + predictors=predictors, + n_estimators=n_estimators, + ) + parts.append(drawn) + return pd.concat(parts).reindex(donor.index) + + +def _flatten(table: Mapping[str, object], prefix: str = "") -> dict[str, float]: + flat: dict[str, float] = {} + for key, value in table.items(): + name = f"{prefix}{key}" + if isinstance(value, Mapping): + flat.update(_flatten(value, prefix=f"{name}.")) + elif isinstance(value, (int, float)): + flat[name] = float(value) + return flat + + +def fit_metrics( + observed: Mapping[str, object], drawn: Mapping[str, object] +) -> dict[str, float]: + """Distributional distances between two fit tables (pp = percentage points).""" + + def share_mae(section: str) -> float: + groups = observed[section] + return float( + np.nanmean( + [ + abs( + drawn[section][g]["share_with_federal_education_debt"] + - groups[g]["share_with_federal_education_debt"] + ) + for g in groups + ] + ) + * 100 + ) + + def tvd(a: Mapping[str, float], b: Mapping[str, float]) -> float: + return 0.5 * float(sum(abs(a[k] - b[k]) for k in a)) + + households_o = observed["households"] + households_d = drawn["households"] + return { + "share_with_debt_error_pp": 100 + * ( + households_d["share_with_federal_education_debt"] + - households_o["share_with_federal_education_debt"] + ), + "mean_among_borrowers_error_pct": 100 + * ( + households_d["mean_among_borrowers"] / households_o["mean_among_borrowers"] + - 1 + ), + "median_among_borrowers_error_pct": 100 + * ( + households_d["median_among_borrowers"] + / households_o["median_among_borrowers"] + - 1 + ), + "household_balance_band_tvd": tvd( + observed["household_balance_bands"], drawn["household_balance_bands"] + ), + "head_age_band_share_mae_pp": share_mae("by_head_age_band"), + "race_share_mae_pp": share_mae("by_race"), + "income_quintile_share_mae_pp": share_mae("by_income_quintile"), + "borrower_balance_band_tvd": tvd( + observed["borrowers"]["balance_bands"], drawn["borrowers"]["balance_bands"] + ), + "borrower_age_band_tvd": tvd( + observed["borrowers"]["age_bands"], drawn["borrowers"]["age_bands"] + ), + } + + +def _rank_correlation(a: np.ndarray, b: np.ndarray) -> float: + ranks_a = pd.Series(a).rank().to_numpy() + ranks_b = pd.Series(b).rank().to_numpy() + return float(np.corrcoef(ranks_a, ranks_b)[0, 1]) + + +def load_host_frame(path: Path) -> Frame: + """Read the columns the stage needs from a Microcosm US H5 (table format).""" + + from microcosm.build.us_runtime.scf_education_debt import ( + _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS, + ) + + with pd.HDFStore(path, "r") as store: + available = set(store.get_storer("/person").data_columns) + columns = [c for c in _HOST_PERSON_COLUMNS if c in available] + [ + c for c in _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS if c in available + ] + person = store.select("/person", columns=columns) + household = store.select( + "/household", columns=["household_id", "household_weight"] + ) + tables = { + "person": person.reset_index(drop=True), + "household": household.loc[:, ["household_id"]].reset_index(drop=True), + } + for entity in ("tax_unit", "spm_unit", "family", "marital_unit"): + tables[entity] = store.select( + f"/{entity}", columns=[f"{entity}_id"] + ).reset_index(drop=True) + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + values=household["household_weight"].to_numpy(dtype=np.float64), + kind=WeightKind.CALIBRATED, + ) + }, + ) + + +def _pct(value: float) -> str: + return "" if value != value else f"{100 * value:.1f}%" + + +def _dollars(value: float) -> str: + return "" if value != value else f"${value:,.0f}" + + +def markdown(receipt: Mapping[str, object]) -> str: + """Render the receipt's tables as the committed Markdown note.""" + + observed = receipt["scf_2025_observed"] + holdout = receipt["holdout"]["variants"][PRODUCTION_VARIANT]["fit"] + host = receipt.get("host", {}).get("fit") + columns = [("SCF 2025", observed), ("SCF hold-out draw", holdout)] + if host is not None: + columns.append((f"Imputed onto {receipt['host']['release']}", host)) + header = "| | " + " | ".join(label for label, _ in columns) + " |" + rule = "|---|" + "---:|" * len(columns) + + def row(label: str, getter, formatter) -> str: + return ( + f"| {label} | " + + " | ".join(formatter(getter(table)) for _, table in columns) + + " |" + ) + + lines = [ + "# microcosm#1073 receipts: SCF 2025 federal student loan balances", + "", + f"Generated {receipt['generated_at']} by " + "`experiments/build_1073_scf_education_debt_receipt.py` on branch " + "`scf-2025-education-debt`. Machine-readable copy: " + "`experiments/receipts/1073-scf-education-debt-fit.json`.", + "", + "Every figure is weighted. SCF rows pool the five implicates with `wgt` " + "(X42001/5). Household figures group by the reference person's age and " + "race and by weighted quintiles of household income less capital gains, " + "the income concept the imputation conditions on. Borrower figures count " + "people with a federal balance (own plus Parent PLUS), in Federal Student " + "Aid's balance and age bands; SCF borrowers are the reference person and " + "spouse. Nothing here is a gate or a calibration target.", + "", + "## Source checks", + "", + ] + checks = receipt["source_checks"] + lines += [ + "| Check | Loader | Fed Alternate Table 13 |", + "|---|---:|---:|", + "| Families with any education debt | " + f"{checks['loader_share_of_families_with_education_debt_pct']:.2f}% | " + f"{FED_ALT_TABLE_13['share_of_families_with_education_debt_pct']:.2f}% |", + "| Median education debt among them | " + f"{_dollars(checks['loader_median_among_holders'])} | " + f"{_dollars(FED_ALT_TABLE_13['median_among_holders'])} |", + "", + "| Share of education-loan balances | Value |", + "|---|---:|", + ] + for key, value in checks["balance_shares"].items(): + lines.append(f"| {key.replace('_', ' ')} | {_pct(value)} |") + lines += [ + "", + "| Federal balances on loans 1-6, by whose education | Share |", + "|---|---:|", + ] + for key, value in checks["federal_balance_share_by_whose_education_code"].items(): + lines.append(f"| {key} | {_pct(value)} |") + lines += [ + "", + "## Fit", + "", + header, + rule, + row( + "Households with federal education debt", + lambda t: t["households"]["share_with_federal_education_debt"], + _pct, + ), + row( + "Mean among them", + lambda t: t["households"]["mean_among_borrowers"], + _dollars, + ), + row( + "Median among them", + lambda t: t["households"]["median_among_borrowers"], + _dollars, + ), + row( + "Weighted federal education debt", + lambda t: t["households"]["federal_education_debt_total"], + lambda v: f"${v / 1e12:,.3f}T", + ), + row( + "Parent PLUS share of the total", + lambda t: t["households"]["parent_plus_share_of_total"], + _pct, + ), + row( + "Borrowers (people)", + lambda t: t["borrowers"]["borrower_weight"], + lambda v: f"{v / 1e6:,.1f}M", + ), + row( + "Mean balance per borrower", + lambda t: t["borrowers"]["mean_balance"], + _dollars, + ), + row( + "Median balance per borrower", + lambda t: t["borrowers"]["median_balance"], + _dollars, + ), + row( + "IBR new borrowers among borrowers", + lambda t: t["borrowers"]["ibr_new_borrower_share"], + _pct, + ), + row( + "Graduate share of own original principal", + lambda t: t["borrowers"]["graduate_share_of_own_original_principal"], + _pct, + ), + "", + "### Borrowers by FSA balance band", + "", + header, + rule, + ] + for label, _, _ in SCF_EDUCATION_DEBT_BALANCE_BANDS: + lines.append( + row(label, lambda t, k=label: t["borrowers"]["balance_bands"][k], _pct) + ) + lines += ["", "### Borrowers by FSA age band", "", header, rule] + for label, _, _ in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: + lines.append( + row(label, lambda t, k=label: t["borrowers"]["age_bands"][k], _pct) + ) + for section, title in ( + ( + "by_head_age_band", + "Households with federal education debt, by age of reference person", + ), + ("by_race", "Households with federal education debt, by race (`racecl4`)"), + ( + "by_income_quintile", + "Households with federal education debt, by income quintile", + ), + ): + lines += ["", f"### {title}", "", header, rule] + for group in observed[section]: + lines.append( + row( + group, + lambda t, s=section, g=group: t[s][g][ + "share_with_federal_education_debt" + ], + _pct, + ) + ) + lines += ["", "Median among borrowers:", "", header, rule] + for group in observed[section]: + lines.append( + row( + group, + lambda t, s=section, g=group: t[s][g]["median_among_borrowers"], + _dollars, + ) + ) + lines += [ + "", + "## Predictor ablation (five-fold hold-out on SCF families)", + "", + "Each variant refits both QRFs. Errors are drawn minus observed; TVD is " + "total variation distance between band distributions; MAE averages the " + "absolute error of the share with debt across the groups. `educ` is the " + "SCF reference person's education, which the CPS side cannot supply until " + "microcosm#297, so that row shows what the predictor would add.", + "", + ] + metrics = [ + "share_with_debt_error_pp", + "mean_among_borrowers_error_pct", + "median_among_borrowers_error_pct", + "household_balance_band_tvd", + "head_age_band_share_mae_pp", + "race_share_mae_pp", + "income_quintile_share_mae_pp", + "borrower_balance_band_tvd", + "borrower_age_band_tvd", + "household_rank_correlation", + ] + lines.append("| Variant | " + " | ".join(metrics) + " |") + lines.append("|---|" + "---:|" * len(metrics)) + for name, variant in receipt["holdout"]["variants"].items(): + values = variant["metrics"] + marker = " (build)" if name == PRODUCTION_VARIANT else "" + lines.append( + f"| {name}{marker} | " + + " | ".join(f"{values[m]:.3f}" for m in metrics) + + " |" + ) + if host is not None: + lines += [ + "", + "## Host fit", + "", + f"Imputed with build seed {receipt['host']['seed']} onto " + f"`{receipt['host']['release']}` ({receipt['host']['households']:,} " + f"households, sha256 `{receipt['host']['sha256'][:12]}...`) at its " + "calibrated household weights; the published weights were calibrated " + "without these columns. Signal gate: " + f"{'passed' if receipt['host']['gate_passed'] else 'FAILED'}.", + "", + "| Metric (imputed minus SCF 2025) | Value |", + "|---|---:|", + ] + for key, value in receipt["host"]["metrics_vs_scf"].items(): + lines.append(f"| {key} | {value:.3f} |") + lines.append("") + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--scf-zip-dir", + type=Path, + help="Directory holding scfp2025s.zip and scf2025s.zip (else fetch).", + ) + parser.add_argument( + "--scf-cache-dir", + type=Path, + default=Path.home() / ".cache" / "microcosm" / "scf", + ) + parser.add_argument("--h5", type=Path, help="US Microcosm H5 to impute onto.") + parser.add_argument("--host-release", default=DEFAULT_HOST_RELEASE) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--folds", type=int, default=5) + parser.add_argument("--n-estimators", type=int, default=100) + parser.add_argument( + "--variants", + nargs="*", + default=list(VARIANTS), + choices=list(VARIANTS), + ) + parser.add_argument( + "--out-json", + type=Path, + default=Path("experiments/receipts/1073-scf-education-debt-fit.json"), + ) + parser.add_argument( + "--out-md", + type=Path, + default=Path("experiments/1073-scf-education-debt-receipts.md"), + ) + args = parser.parse_args(argv) + + if args.scf_zip_dir is not None: + summary_path = _member_from_local_zip( + SCF_2025_SUMMARY_EXTRACT, args.scf_zip_dir.expanduser(), args.scf_cache_dir + ) + full_path = _member_from_local_zip( + SCF_2025_FULL_EXTRACT, args.scf_zip_dir.expanduser(), args.scf_cache_dir + ) + else: + summary_path = fetch_scf_public_file( + SCF_2025_SUMMARY_EXTRACT, args.scf_cache_dir + ) + full_path = fetch_scf_public_file(SCF_2025_FULL_EXTRACT, args.scf_cache_dir) + + donor = load_scf_2025_education_debt_donor(summary_path, full_path) + full = _read_stata_columns( + full_path, + ( + "y1", + "yy1", + "x42001", + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + ), + label="2025 full file", + ) + full["wgt"] = full["x42001"] / 5.0 + observed = scf_education_debt_donor_fit_table(donor) + receipt: dict[str, object] = { + "schema_version": 1, + "issue": ISSUE, + "generated_at": datetime.now(UTC).strftime("%Y-%m-%d"), + "scf_files": { + public_file.member: { + "url": public_file.url, + "zip_sha256": public_file.zip_sha256, + "member_sha256": public_file.member_sha256, + } + for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT) + }, + "source_checks": source_checks(donor, full), + "scf_2025_observed": observed, + "holdout": { + "folds": args.folds, + "seed": args.seed, + "n_estimators": args.n_estimators, + "production_variant": PRODUCTION_VARIANT, + "variants": {}, + }, + } + observed_total = donor[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(dtype=np.float64) + for name in args.variants: + predictors = VARIANTS[name] + drawn = holdout_draws( + donor, + predictors, + folds=args.folds, + seed=args.seed, + n_estimators=args.n_estimators, + ) + fit = scf_education_debt_donor_fit_table(donor, imputed=drawn) + metrics = fit_metrics(observed, fit) + metrics["household_rank_correlation"] = _rank_correlation( + observed_total, + drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(dtype=np.float64), + ) + receipt["holdout"]["variants"][name] = { + "predictors": list(predictors), + "fit": fit, + "metrics": metrics, + } + print(name, json.dumps(metrics, indent=None, default=float)) + + if args.h5 is not None: + host_sha256 = _sha256(args.h5) + frame = load_host_frame(args.h5) + frame = with_us_scf_education_debt_inputs( + frame, + seed=args.seed, + time_period=2024, + scf_education_debt_donor=donor, + n_estimators=args.n_estimators, + ) + gate = us_scf_education_debt_signal_gate(frame) + host_fit = us_scf_education_debt_fit_table(frame) + receipt["host"] = { + "release": args.host_release, + "path": args.h5.name, + "sha256": host_sha256, + "is_the_published_default": host_sha256 == DEFAULT_HOST_SHA256, + "households": int(len(frame.table("household"))), + "people": int(len(frame.table("person"))), + "seed": args.seed, + "gate_passed": bool(gate.passed), + "gate_failures": list(gate.failures), + "household_income_columns_present": gate.details.get( + "household_income_columns_present" + ), + "fit": host_fit, + "metrics_vs_scf": fit_metrics(observed, host_fit), + } + print("host", json.dumps(receipt["host"]["metrics_vs_scf"], default=float)) + + args.out_json.parent.mkdir(parents=True, exist_ok=True) + args.out_json.write_text( + json.dumps(receipt, indent=2, sort_keys=False, default=float) + "\n", + encoding="utf-8", + newline="\n", + ) + args.out_md.write_text(markdown(receipt), encoding="utf-8", newline="\n") + print(f"wrote {args.out_json} and {args.out_md}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 72b56c12ab..956a7dcccf 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -855,7 +855,7 @@ }, { "stage": "scf_wealth", - "survey": "Fed SCF 2022 + Census SIPP 2023", + "survey": "Fed SCF 2022 + Fed SCF 2025 + Census SIPP 2023", "source": "https://www.federalreserve.gov/econres/scfindex.htm", "grain": "household", "artifacts": [ @@ -895,6 +895,26 @@ "revision": "21280dca5995e978d706740a8a4b9b7860cfd7b6", "sha256": "5c30439e365fc26483318ef61d1d8f4bb2f0e9d6bb47c22c06756a7698733ee2", "size_bytes": 3726010471 + }, + { + "kind": "public_microdata", + "format": "stata_in_zip", + "vintage": "2025", + "locator": "https://www.federalreserve.gov/econres/files/scfp2025s.zip", + "member": "rscfp2025.dta", + "sha256": "32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + "member_sha256": "454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd", + "expected_rows": 21800 + }, + { + "kind": "public_microdata", + "format": "stata_in_zip", + "vintage": "2025", + "locator": "https://www.federalreserve.gov/econres/files/scf2025s.zip", + "member": "p25i6.dta", + "sha256": "c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + "member_sha256": "38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad", + "expected_rows": 21800 } ], "operations": [ @@ -1149,6 +1169,151 @@ "stock_assets" ] }, + { + "kind": "read_tables", + "tables": [ + "scf_2025_summary_extract", + "scf_2025_full_extract" + ], + "weight": "wgt" + }, + { + "kind": "join", + "left": "scf_2025_summary_extract", + "right": "scf_2025_full_extract", + "keys": [ + "y1", + "yy1" + ], + "validation": "one_to_one_all_five_implicates" + }, + { + "kind": "derive", + "outputs": [ + "federal_education_debt", + "head_federal_student_loan_balance", + "spouse_federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower" + ], + "method": "scf_federal_education_loans_by_holder", + "inputs": [ + "x7824", + "x7847", + "x7870", + "x7924", + "x7947", + "x7970", + "x7179", + "x7879", + "x7884", + "x7889", + "x7894", + "x7899", + "x7994", + "x7978", + "x7883", + "x7888", + "x7893", + "x7898", + "x7993", + "x9332", + "x9333", + "x9334", + "x9335", + "x9336", + "x9337", + "x9338", + "x9339", + "x9340", + "x9341", + "x9342", + "x9343", + "x7805", + "x7828", + "x7851", + "x7905", + "x7928", + "x7951" + ], + "allocation": "x7179_in_proportion_to_loans_1_6", + "source": "scf_2025_full_extract" + }, + { + "kind": "fit_weighted_qrf", + "donor": "scf_2025_education_debt", + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "targets": [ + "federal_education_debt" + ], + "weight": "wgt", + "n_estimators": 100, + "seed_from_build_config": true + }, + { + "kind": "fit_weighted_qrf", + "donor": "scf_2025_education_debt_borrowers", + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income", + "federal_education_debt" + ], + "targets": [ + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower" + ], + "weight": "wgt", + "response_filter": "federal_education_debt > 0", + "n_estimators": 100, + "seed_from_build_config": true + }, + { + "kind": "derive", + "outputs": [ + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share" + ], + "method": "allocate_household_education_debt_to_reference_person_and_spouse", + "inputs": [ + "federal_education_debt", + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower", + "A_SPOUSE", + "PECOHAB" + ] + }, { "kind": "support_clip", "range": "donor_realized" @@ -1180,7 +1345,11 @@ "qualified_passenger_vehicle_loan_interest", "bank_account_assets", "bond_assets", - "stock_assets" + "stock_assets", + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share" ], "nonnegative_outputs": [ "auto_loan_balance", @@ -1188,9 +1357,12 @@ "qualified_passenger_vehicle_loan_interest", "bank_account_assets", "bond_assets", - "stock_assets" + "stock_assets", + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "federal_student_loan_graduate_share" ], - "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped." + "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family's federal share and holder split of loans 1-6. Whose education: code 1 (the reference person) and code 2 (spouse or partner) are each person's own federal_student_loan_balance; code 3 (child, with grandchild and other relative folded in by the public file) is the reference person's federal_parent_plus_loan_balance; codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the reference person's own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower is true when every federal loan the person holds was borrowed in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person's own federal loans that financed a master's, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all families on the eight shared predictors plus household income less capital gains; a second, fit on borrower families and conditioning on the total, draws the chained holder shares, graduate shares and new-borrower flags. The SCF-style reference person and the member named by its A_SPOUSE (else PECOHAB) pointer receive the balances; without one present the spouse's draw goes to the reference person. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals." }, { "stage": "ssi_disability_criteria", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml index 128371e097..b73b86ef0e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml @@ -278,6 +278,9 @@ tail_contracts: - farm_operations_income_would_be_qualified - farm_rent_income - farm_rent_income_would_be_qualified + - federal_parent_plus_loan_balance + - federal_student_loan_balance + - federal_student_loan_graduate_share - first_home_mortgage_balance - first_home_mortgage_interest - first_home_mortgage_origination_year @@ -292,6 +295,7 @@ tail_contracts: - is_executive_administrative_professional - is_farmer_fisher - is_hispanic + - is_income_based_repayment_new_borrower - is_military - is_paid_hourly - is_union_member_or_covered diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index e33ffec848..5711f25977 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf + sha256: c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720 stage_manifest: version: 1 country: us @@ -954,7 +954,7 @@ stages: nonnegative_outputs: - schedule_d_capital_gain_distributions - stage: scf_wealth - survey: Fed SCF 2022 + Census SIPP 2023 + survey: Fed SCF 2022 + Fed SCF 2025 + Census SIPP 2023 source: https://www.federalreserve.gov/econres/scfindex.htm grain: household artifacts: @@ -989,6 +989,22 @@ stages: revision: 21280dca5995e978d706740a8a4b9b7860cfd7b6 sha256: 5c30439e365fc26483318ef61d1d8f4bb2f0e9d6bb47c22c06756a7698733ee2 size_bytes: 3726010471 + - kind: public_microdata + format: stata_in_zip + vintage: '2025' + locator: https://www.federalreserve.gov/econres/files/scfp2025s.zip + member: rscfp2025.dta + sha256: 32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb + member_sha256: 454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd + expected_rows: 21800 + - kind: public_microdata + format: stata_in_zip + vintage: '2025' + locator: https://www.federalreserve.gov/econres/files/scf2025s.zip + member: p25i6.dta + sha256: c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741 + member_sha256: 38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad + expected_rows: 21800 operations: - kind: read_tables tables: @@ -1200,6 +1216,129 @@ stages: - bank_account_assets - bond_assets - stock_assets + - kind: read_tables + tables: + - scf_2025_summary_extract + - scf_2025_full_extract + weight: wgt + - kind: join + left: scf_2025_summary_extract + right: scf_2025_full_extract + keys: + - y1 + - yy1 + validation: one_to_one_all_five_implicates + - kind: derive + outputs: + - federal_education_debt + - head_federal_student_loan_balance + - spouse_federal_student_loan_balance + - federal_parent_plus_loan_balance + - head_own_share + - spouse_own_share + - head_graduate_share + - spouse_graduate_share + - head_is_ibr_new_borrower + - spouse_is_ibr_new_borrower + method: scf_federal_education_loans_by_holder + inputs: + - x7824 + - x7847 + - x7870 + - x7924 + - x7947 + - x7970 + - x7179 + - x7879 + - x7884 + - x7889 + - x7894 + - x7899 + - x7994 + - x7978 + - x7883 + - x7888 + - x7893 + - x7898 + - x7993 + - x9332 + - x9333 + - x9334 + - x9335 + - x9336 + - x9337 + - x9338 + - x9339 + - x9340 + - x9341 + - x9342 + - x9343 + - x7805 + - x7828 + - x7851 + - x7905 + - x7928 + - x7951 + allocation: x7179_in_proportion_to_loans_1_6 + source: scf_2025_full_extract + - kind: fit_weighted_qrf + donor: scf_2025_education_debt + predictors: + - age + - is_female + - cps_race + - is_married + - own_children_in_household + - employment_income + - interest_dividend_income + - social_security_pension_income + - household_income + targets: + - federal_education_debt + weight: wgt + n_estimators: 100 + seed_from_build_config: true + - kind: fit_weighted_qrf + donor: scf_2025_education_debt_borrowers + predictors: + - age + - is_female + - cps_race + - is_married + - own_children_in_household + - employment_income + - interest_dividend_income + - social_security_pension_income + - household_income + - federal_education_debt + targets: + - head_own_share + - spouse_own_share + - head_graduate_share + - spouse_graduate_share + - head_is_ibr_new_borrower + - spouse_is_ibr_new_borrower + weight: wgt + response_filter: federal_education_debt > 0 + n_estimators: 100 + seed_from_build_config: true + - kind: derive + outputs: + - federal_student_loan_balance + - federal_parent_plus_loan_balance + - is_income_based_repayment_new_borrower + - federal_student_loan_graduate_share + method: allocate_household_education_debt_to_reference_person_and_spouse + inputs: + - federal_education_debt + - head_own_share + - spouse_own_share + - head_graduate_share + - spouse_graduate_share + - head_is_ibr_new_borrower + - spouse_is_ibr_new_borrower + - A_SPOUSE + - PECOHAB - kind: support_clip range: donor_realized outputs: @@ -1229,6 +1368,10 @@ stages: - bank_account_assets - bond_assets - stock_assets + - federal_student_loan_balance + - federal_parent_plus_loan_balance + - is_income_based_repayment_new_borrower + - federal_student_loan_graduate_share nonnegative_outputs: - auto_loan_balance - auto_loan_interest @@ -1236,6 +1379,9 @@ stages: - bank_account_assets - bond_assets - stock_assets + - federal_student_loan_balance + - federal_parent_plus_loan_balance + - federal_student_loan_graduate_share notes: 'Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45''s final 13-predictor asset QRF, while @@ -1251,7 +1397,26 @@ stages: leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS''s approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth - is never clipped.' + is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output + stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 + = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), + the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal + loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond + the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans + 1-6): X7179 takes the family''s federal share and holder split of loans 1-6. Whose education: code + 1 (the reference person) and code 2 (spouse or partner) are each person''s own federal_student_loan_balance; + code 3 (child, with grandchild and other relative folded in by the public file) is the reference person''s + federal_parent_plus_loan_balance; codes 6 (multiple people), 7 (unrelated person) and -7 (other) are + the reference person''s own balance, since none can be a Parent PLUS loan for one dependent student. + is_income_based_repayment_new_borrower is true when every federal loan the person holds was borrowed + in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share + is the amount-borrowed-weighted share of the person''s own federal loans that financed a master''s, + professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all + families on the eight shared predictors plus household income less capital gains; a second, fit on + borrower families and conditioning on the total, draws the chained holder shares, graduate shares + and new-borrower flags. The SCF-style reference person and the member named by its A_SPOUSE (else + PECOHAB) pointer receive the balances; without one present the spouse''s draw goes to the reference + person. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals.' - stage: ssi_disability_criteria survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml index a5b70b9085..fc2ca10fad 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml @@ -110,6 +110,15 @@ records: field: vintage compatible_with: - vintage:target_2024 +- id: scf_2025 + kind: survey_period_ref + authority_ref: + kind: source_stage_artifact + stage: scf_wealth + artifact_index: 3 + field: vintage + compatible_with: + - vintage:target_2024 - id: sipp_2023 kind: survey_period_ref authority_ref: @@ -134,6 +143,7 @@ records: - vintage:policyengine_us_surface - vintage:release_us_2024 - vintage:scf_2022 + - vintage:scf_2025 - vintage:sipp_2023 - vintage:tax_2015 - id: tax_2015 diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 74e4757cba..4063aadbe5 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -764,6 +764,25 @@ us_scf_auto_loans_summary, with_us_scf_auto_loan_inputs, ) +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_VINTAGE, + SCF_WHOSE_EDUCATION_HOLDER, + US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS, + US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, + derive_scf_education_debt_targets, + fetch_scf_public_file, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, + us_scf_education_debt_stage_spec, + us_scf_education_debt_summary, + with_us_scf_education_debt_inputs, +) from microcosm.build.us_runtime.scf_wealth import ( FINANCIAL_ASSET_BLEND_AUDIT_KEY, FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY, @@ -1575,6 +1594,23 @@ "us_scf_auto_loans_stage_spec", "us_scf_auto_loans_summary", "with_us_scf_auto_loan_inputs", + "SCF_2025_FULL_EXTRACT", + "SCF_2025_SUMMARY_EXTRACT", + "SCF_EDUCATION_DEBT_PREDICTORS", + "SCF_EDUCATION_DEBT_VINTAGE", + "SCF_WHOSE_EDUCATION_HOLDER", + "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", + "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS", + "derive_scf_education_debt_targets", + "fetch_scf_public_file", + "impute_us_scf_education_debt", + "load_scf_2025_education_debt_donor", + "scf_education_debt_donor_fit_table", + "us_scf_education_debt_fit_table", + "us_scf_education_debt_signal_gate", + "us_scf_education_debt_stage_spec", + "us_scf_education_debt_summary", + "with_us_scf_education_debt_inputs", "FINANCIAL_ASSET_BLEND_AUDIT_KEY", "FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY", "SCF_FINANCIAL_ASSET_TARGET_COMPONENTS", @@ -2082,13 +2118,14 @@ def to_manifest(self) -> dict[str, object]: #: the observatory's sources diagram and the dataset card derive from it. US_DONORS: Mapping[str, DonorSpec] = { "scf_wealth": DonorSpec( - survey="Fed SCF 2022 + Census SIPP 2023", + survey="Fed SCF 2022 + Fed SCF 2025 + Census SIPP 2023", source="https://www.federalreserve.gov/econres/scfindex.htm", notes=( "SCF anchors signed net worth and one half of household liquid-asset " "vectors; the immutable SIPP 2023 public-use donor supplies the " "other half and restores low liquid-asset mass. Auto loans use the " - "full SCF separately." + "full SCF separately. Federal education debt reads the 2025 SCF " + "(microcosm#1073) while every other output stays on 2022." ), ), US_SSI_DISABILITY_CRITERIA_STAGE_NAME: DonorSpec( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py new file mode 100644 index 0000000000..747ccfc99e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -0,0 +1,1611 @@ +"""Federal student loan balances from the 2025 Survey of Consumer Finances. + +microcosm#1073 carries SCF education debt into the US build for the EqRC student +debt pilot (PolicyEngine/us-student-debt#1). It writes four person inputs of +PolicyEngine-US: ``federal_student_loan_balance`` and +``federal_parent_plus_loan_balance`` (policyengine-us#9721), and +``is_income_based_repayment_new_borrower`` and +``federal_student_loan_graduate_share`` (policyengine-us#9724). + +**Vintage.** Only education debt reads the 2025 SCF. Net worth, the liquid-asset +leaves and auto loans stay on the 2022 SCF in :mod:`scf_wealth` and +:mod:`scf_auto_loans` until the whole ``scf_wealth`` stage moves. The 2025 wave +is three years closer to the build, follows the end of the payment pause, and +records each loan's year borrowed and program type, which 2022 does not. Every +2025-specific coordinate lives in :data:`SCF_2025_SUMMARY_EXTRACT` and +:data:`SCF_2025_FULL_EXTRACT`, and the predictor construction is the shared +summary-extract one from :mod:`scf_wealth`. The 2025 summary extract carries +every column that construction reads, so moving the stage is a change of pinned +coordinates, not of code. + +**Donor targets, per SCF family and implicate.** For each of education loans +1-6 the full public file records the amount still owed (X7824, X7847, X7870, +X7924, X7947, X7970), whether it is a federal loan (X7879, X7884, X7889, X7894, +X7899, X7994; 1 = yes), whose education it paid for (X7978, X7883, X7888, +X7893, X7898, X7993), the year borrowed (X9332-X9337), the program it financed +(X9338-X9343) and the amount borrowed (X7805, X7828, X7851, X7905, X7928, +X7951). Loans beyond the sixth are one total, X7179, with none of these items. + +- Only loans flagged federal (code 1) count. A loan whose federal status is + not recorded (code 0; the 2025 public file leaves some "don't know" answers + blank) counts as non-federal. +- **X7179 rule.** The remaining-loans total is allocated in proportion to the + family's balances on loans 1-6: every loan 1-6 is scaled by + ``1 + X7179 / (sum of balances on loans 1-6)``. This applies the family's + federal share of loans 1-6 to X7179, and splits the federal part among people + in the same proportions as loans 1-6. A family with X7179 but no balance on + loans 1-6 contributes nothing (none in the public files read). +- **Whose-education rule** (:data:`SCF_WHOSE_EDUCATION_HOLDER`). Code 1 + (respondent; the public file swaps respondent and spouse codes so code 1 is + the SCF reference person) is the head's own loan, and code 2 (spouse or + partner) the spouse's own loan; both go to ``federal_student_loan_balance``. + Code 3 (child or stepchild; the public file folds grandchild and other + relative into 3) is a loan for a child's education and goes to the head's + ``federal_parent_plus_loan_balance``. Code 6 (multiple people), code 7 + (unrelated person) and code -7 (other) go to the head's own balance: none can + be a Parent PLUS loan for one dependent student, and the SCF reference person + is the household's default borrower. Any other code (the public file folds 4 + and 5 into 3, so none is expected) also goes to the head. +- **IBR new-borrower status** is true when the person holds a federal loan and + every federal loan the person holds (own and Parent PLUS) was borrowed in + 2014 or later. The item records a year only, so 2014 counts as after July 1, + 2014. For a consolidated loan the year is that of the most recent underlying + loan. +- **Graduate share** is the share of the original amount of the person's own + federal loans (Parent PLUS excluded) that financed a master's, professional + or doctoral program (codes 13-15), weighted by the amount borrowed (falling + back to the balance when the amount is missing), as 34 CFR 685.209(f)(1) + weights the SAVE rates. + +**Imputation.** Two regime-gated weighted QRFs (:class:`microcosm.fit.QRF`) +condition on the eight shared SCF predictors plus household income excluding +capital gains (:data:`SCF_EDUCATION_DEBT_PREDICTORS`). The first draws each +recipient household's federal education debt from every SCF family. The +second, fit on borrower families only and conditioning also on that total, +draws the chained composition: the head's and spouse's shares of the total, +their graduate shares, and their new-borrower flags. Shares are clipped to +[0, 1] and rescaled when the two own shares exceed one; the remainder is the +head's Parent PLUS balance. + +**Recipients.** The household's SCF-style reference person +(:func:`scf_auto_loans._scf_reference_person_mask`) carries the reference +person's demographics and the household-summed incomes, as for auto loans. The +spouse is the member whose line number is the reference person's ``A_SPOUSE``, +or else its ``PECOHAB`` partner when that column is present. A household drawn +with a spouse's balance but no spouse or partner present gives that balance to +the reference person, whose graduate share becomes the balance-weighted mean +of the two and whose new-borrower flag requires both. Everyone else gets zero. + +**Known limits.** The SCF records the debts of the primary economic unit, so a +financially independent adult living in someone else's household is missed, +and an adult child's own loan in the unit is coded 3 and lands on the parent as +Parent PLUS. The SCF federal flag includes FFEL and Perkins loans that +``federal_student_loan_balance`` excludes until consolidated. Balances are +2025 survey-date dollars. The FSA calibration (microcosm#1074) pins totals. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.gates import GateResult +from microcosm.build.source_manifest import SourceStageSpec, load_source_manifest +from microcosm.build.us_runtime.eligibility_inputs import _own_children_in_household +from microcosm.build.us_runtime.scf_auto_loans import _scf_reference_person_mask +from microcosm.build.us_runtime.scf_wealth import ( + SCF_WEALTH_PREDICTORS, + _recipient_cps_race, + _replace_sentinels, + _scf_summary_predictor_table, + _sum_present, +) +from microcosm.frame import Frame +from microcosm.frame.units import US_SCHEMA + +__all__ = [ + "SCF_2025_FULL_EXTRACT", + "SCF_2025_SUMMARY_EXTRACT", + "SCF_EDUCATION_DEBT_BALANCE_BANDS", + "SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS", + "SCF_EDUCATION_DEBT_COMPOSITION_TARGETS", + "SCF_EDUCATION_DEBT_PREDICTORS", + "SCF_EDUCATION_DEBT_TOTAL_TARGET", + "SCF_EDUCATION_DEBT_VINTAGE", + "SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS", + "SCF_EDUCATION_LOAN_BALANCE_COLUMNS", + "SCF_EDUCATION_LOAN_FEDERAL_COLUMNS", + "SCF_EDUCATION_LOAN_PROGRAM_COLUMNS", + "SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN", + "SCF_EDUCATION_LOAN_WHOSE_COLUMNS", + "SCF_EDUCATION_LOAN_YEAR_COLUMNS", + "SCF_WHOSE_EDUCATION_HOLDER", + "ScfPublicFile", + "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", + "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS", + "derive_scf_education_debt_targets", + "draw_scf_education_debt_households", + "education_debt_fit_table", + "fetch_scf_public_file", + "impute_us_scf_education_debt", + "load_scf_2025_education_debt_donor", + "scf_education_debt_donor_fit_table", + "us_scf_education_debt_fit_table", + "us_scf_education_debt_signal_gate", + "us_scf_education_debt_stage_spec", + "us_scf_education_debt_summary", + "with_us_scf_education_debt_inputs", +] + +_SCF_STAGE_NAME = "scf_wealth" +_DONOR_WEIGHT_COLUMN = "scf_weight" +_HOUSEHOLD_ID_COLUMN = "person_household_id" +_DEFAULT_N_ESTIMATORS = 100 +_SCF_HTTP_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" +) + +#: The SCF wave education debt reads (``scf_wealth`` itself stays on 2022). +SCF_EDUCATION_DEBT_VINTAGE = "2025" + + +@dataclass(frozen=True) +class ScfPublicFile: + """One pinned Federal Reserve SCF public file: a Stata member in a zip.""" + + url: str + member: str + zip_sha256: str + member_sha256: str + rows: int + + @property + def zip_name(self) -> str: + return self.url.rsplit("/", maxsplit=1)[-1] + + +#: The 2025 public summary extract. Both 2025 files were released 2026-10-09; +#: these digests match a fresh Federal Reserve download of 2026-10-12. Each +#: file has one row per family and implicate (4,360 families x 5). +SCF_2025_SUMMARY_EXTRACT = ScfPublicFile( + url="https://www.federalreserve.gov/econres/files/scfp2025s.zip", + member="rscfp2025.dta", + zip_sha256="32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + member_sha256="454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd", + rows=21_800, +) +#: The 2025 full public file. +SCF_2025_FULL_EXTRACT = ScfPublicFile( + url="https://www.federalreserve.gov/econres/files/scf2025s.zip", + member="p25i6.dta", + zip_sha256="c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + member_sha256="38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad", + rows=21_800, +) + +#: Education loans 1-6 (codebook "EDUCATION LOANS"): amount still owed. +SCF_EDUCATION_LOAN_BALANCE_COLUMNS: tuple[str, ...] = ( + "x7824", + "x7847", + "x7870", + "x7924", + "x7947", + "x7970", +) +#: Total owed on all remaining loans beyond the sixth. +SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN = "x7179" +#: "Is this loan a federal student loan?" 1 = yes, 5 = no. +SCF_EDUCATION_LOAN_FEDERAL_COLUMNS: tuple[str, ...] = ( + "x7879", + "x7884", + "x7889", + "x7894", + "x7899", + "x7994", +) +#: "For whose education was this loan taken out?" +SCF_EDUCATION_LOAN_WHOSE_COLUMNS: tuple[str, ...] = ( + "x7978", + "x7883", + "x7888", + "x7893", + "x7898", + "x7993", +) +#: "In what year was this loan (or the most recent underlying loan) borrowed?" +SCF_EDUCATION_LOAN_YEAR_COLUMNS: tuple[str, ...] = ( + "x9332", + "x9333", + "x9334", + "x9335", + "x9336", + "x9337", +) +#: "What type of educational program did this loan help finance?" +SCF_EDUCATION_LOAN_PROGRAM_COLUMNS: tuple[str, ...] = ( + "x9338", + "x9339", + "x9340", + "x9341", + "x9342", + "x9343", +) +#: "How much was borrowed for this loan?" (or the balance after the most +#: recent consolidation or refinancing). +SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS: tuple[str, ...] = ( + "x7805", + "x7828", + "x7851", + "x7905", + "x7928", + "x7951", +) +_SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN = "x19" + +_FEDERAL_LOAN_YES = 1 +#: Programs counted as graduate or professional: master's (13), professional +#: school (14) and doctorate (15). +_GRADUATE_PROGRAM_CODES = frozenset({13, 14, 15}) +#: First year counted as a new IBR borrower (20 U.S.C. 1098e(e): no +#: outstanding balance on July 1, 2014). The SCF records only the year. +_IBR_NEW_BORROWER_FIRST_YEAR = 2014 + +_HEAD = "head" +_SPOUSE = "spouse" +_PARENT_PLUS = "parent_plus" +#: Whose-education code -> who holds the loan in the build. See the module +#: docstring for the rule and its reasons. +SCF_WHOSE_EDUCATION_HOLDER: Mapping[int, str] = { + 1: _HEAD, + 2: _SPOUSE, + 3: _PARENT_PLUS, + 6: _HEAD, + 7: _HEAD, + -7: _HEAD, +} + +#: Household-total target of the first QRF. +SCF_EDUCATION_DEBT_TOTAL_TARGET = "federal_education_debt" +#: Chained targets of the second (borrower-family) QRF. +SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: tuple[str, ...] = ( + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower", +) +#: The shared SCF predictors plus household income excluding capital gains. +#: Educational attainment joins once microcosm#297 carries it on the CPS side. +SCF_EDUCATION_DEBT_PREDICTORS: tuple[str, ...] = ( + *SCF_WEALTH_PREDICTORS, + "household_income", +) + +US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS: tuple[str, ...] = ( + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", +) +US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS: tuple[str, ...] = ( + *US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS, + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share", +) + +#: Federal Student Aid portfolio bands (dollars, lower bound inclusive). +SCF_EDUCATION_DEBT_BALANCE_BANDS: tuple[tuple[str, float, float], ...] = ( + ("under_5k", 0.0, 5_000.0), + ("5k_to_10k", 5_000.0, 10_000.0), + ("10k_to_20k", 10_000.0, 20_000.0), + ("20k_to_40k", 20_000.0, 40_000.0), + ("40k_to_60k", 40_000.0, 60_000.0), + ("60k_to_80k", 60_000.0, 80_000.0), + ("80k_to_100k", 80_000.0, 100_000.0), + ("100k_to_200k", 100_000.0, 200_000.0), + ("200k_plus", 200_000.0, np.inf), +) +#: Federal Student Aid borrower age bands (years, inclusive bounds). +SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: tuple[tuple[str, float, float], ...] = ( + ("24_and_younger", -np.inf, 24.0), + ("25_to_34", 25.0, 34.0), + ("35_to_49", 35.0, 49.0), + ("50_to_61", 50.0, 61.0), + ("62_and_older", 62.0, np.inf), +) +#: SCF ``racecl4`` labels (2025 folds Middle Eastern or North African into 1). +_RACECL4_LABELS: Mapping[int, str] = { + 1: "white_non_hispanic", + 2: "black_non_hispanic", + 3: "hispanic", + 4: "other_or_multiple", +} + +#: Summary-extract columns read beyond the shared predictor surface. +_SUMMARY_EXTRA_COLUMNS: tuple[str, ...] = ( + "y1", + "yy1", + "income", + "kginc", + "racecl4", + "edn_inst", + "educ", +) +#: Person income inputs summed into the recipient household's income, the CPS +#: counterpart of SCF ``income`` less ``kginc``. Absent columns count as zero. +_RECIPIENT_HOUSEHOLD_INCOME_COLUMNS: tuple[str, ...] = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "farm_operations_income", + "farm_rent_income", + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "rental_income", + "estate_income", + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents", + "taxable_private_pension_income", + "tax_exempt_private_pension_income", + "taxable_ira_distributions", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_sep_distributions", + "unemployment_compensation", + "alimony_income", + "child_support_received", + "veterans_benefits", + "workers_compensation", + "disability_benefits", +) +_RECIPIENT_INTEREST_DIVIDEND_COLUMNS = ( + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", +) +_RECIPIENT_SS_PENSION_COLUMNS = ( + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents", + "taxable_private_pension_income", + "tax_exempt_private_pension_income", +) +_REQUIRED_RECIPIENT_PERSON_COLUMNS: tuple[str, ...] = ( + _HOUSEHOLD_ID_COLUMN, + "age", + "is_female", + "PRDTRACE", + "PRDTHSP", + "A_MARITL", + "A_LINENO", + "A_SPOUSE", + "PH_SEQ", + "PEPAR1", + "PEPAR2", + "employment_income_before_lsr", +) +_OPTIONAL_PARTNER_COLUMN = "PECOHAB" + +#: Weighted share of households holding federal education debt allowed by the +#: signal gate. The SCF 2025 share holding any education debt is 19.9%. +_BORROWER_HOUSEHOLD_SHARE_BAND = (0.08, 0.35) + + +# --------------------------------------------------------------------------- # +# Source files # +# --------------------------------------------------------------------------- # +def _sha256_hexdigest(payload: bytes) -> str: + import hashlib + + return hashlib.sha256(payload).hexdigest() + + +def fetch_scf_public_file( + public_file: ScfPublicFile, + cache_dir: str | Path | None = None, +) -> Path: + """Download, verify and cache one pinned SCF public Stata member. + + A cached member whose SHA-256 matches the pin is reused without a network + call; anything else is fetched from the Federal Reserve, and both the zip + and the member must match their pins before the member is cached. + """ + + import io + import urllib.request + import zipfile + + root = ( + Path(cache_dir).expanduser() + if cache_dir is not None + else Path.home() / ".cache" / "microcosm" / "scf" + ) + root.mkdir(parents=True, exist_ok=True) + target = root / public_file.member + if target.exists() and target.stat().st_size > 0: + if _sha256_hexdigest(target.read_bytes()) == public_file.member_sha256: + return target + + request = urllib.request.Request( + public_file.url, + headers={"User-Agent": _SCF_HTTP_USER_AGENT}, + ) + with urllib.request.urlopen(request, timeout=300) as response: # noqa: S310 + payload = response.read() + digest = _sha256_hexdigest(payload) + if digest != public_file.zip_sha256: + raise ValueError( + f"SCF {public_file.zip_name} failed sha-256 verification: " + f"expected {public_file.zip_sha256}, got {digest}." + ) + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + if public_file.member not in archive.namelist(): + raise ValueError( + f"SCF {public_file.zip_name} is missing member {public_file.member!r}." + ) + member_bytes = archive.read(public_file.member) + digest = _sha256_hexdigest(member_bytes) + if digest != public_file.member_sha256: + raise ValueError( + f"SCF {public_file.member} failed sha-256 verification: " + f"expected {public_file.member_sha256}, got {digest}." + ) + target.write_bytes(member_bytes) + return target + + +def _read_stata_columns( + path: str | Path, + columns: Sequence[str], + *, + label: str, + chunksize: int = 4_000, +) -> pd.DataFrame: + """Read named columns of a Stata file in row chunks. + + The full public file is about 220 MB with over 5,000 variables; reading it + whole would hold every variable in memory. Chunks keep only the requested + columns resident. + """ + + wanted = list(dict.fromkeys(columns)) + try: + with pd.read_stata( + path, + convert_categoricals=False, + columns=wanted, + chunksize=chunksize, + ) as reader: + parts = [chunk for chunk in reader] + except ValueError as error: + if "not found in the Stata data set" not in str(error): + raise + raise ValueError(f"SCF {label} missing required column(s): {error}.") from error + table = pd.concat(parts, ignore_index=True) if parts else pd.DataFrame() + missing = sorted(set(wanted) - set(table.columns)) + if missing: + raise ValueError(f"SCF {label} missing required column(s): {missing}.") + return table.loc[:, wanted] + + +# --------------------------------------------------------------------------- # +# Donor targets # +# --------------------------------------------------------------------------- # +def _numeric(table: pd.DataFrame, column: str) -> np.ndarray: + return ( + pd.to_numeric(table[column], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + + +def derive_scf_education_debt_targets(full: pd.DataFrame) -> pd.DataFrame: + """Derive household federal education-debt targets from per-loan items. + + Pure and row-preserving: one output row per input (family, implicate) row. + See the module docstring for the federal, X7179, whose-education, IBR and + graduate-share rules this applies. + + Returns columns: + ``federal_education_debt``: federal balance after the X7179 rule. + ``head_federal_student_loan_balance``, + ``spouse_federal_student_loan_balance``, + ``federal_parent_plus_loan_balance``: its split by holder. + ``head_own_share``, ``spouse_own_share``: the two own balances over the + total (zero for a family without federal education debt). + ``head_is_ibr_new_borrower``, ``spouse_is_ibr_new_borrower``: 0/1. + ``head_graduate_share``, ``spouse_graduate_share``: in [0, 1]. + ``education_debt``: every education loan, federal or not (EDN_INST + without the education-coded installment loans). + ``federal_status_unrecorded_balance``: loans 1-6 balances whose federal + flag is blank (counted as non-federal). + ``remaining_loans_without_detail``: X7179 in a family with no balance + on loans 1-6 (dropped). + """ + + required = ( + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + *SCF_EDUCATION_LOAN_YEAR_COLUMNS, + *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + ) + missing = [column for column in required if column not in full.columns] + if missing: + raise ValueError(f"SCF education-debt derivation missing column(s): {missing}.") + + n = len(full) + balances = np.column_stack( + [np.maximum(_numeric(full, c), 0.0) for c in SCF_EDUCATION_LOAN_BALANCE_COLUMNS] + ).reshape(n, len(SCF_EDUCATION_LOAN_BALANCE_COLUMNS)) + remaining = np.maximum( + _numeric(full, SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN), 0.0 + ) + federal_code = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_FEDERAL_COLUMNS] + ).reshape(balances.shape) + whose = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_WHOSE_COLUMNS] + ).reshape(balances.shape) + year = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_YEAR_COLUMNS] + ).reshape(balances.shape) + program = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_PROGRAM_COLUMNS] + ).reshape(balances.shape) + borrowed = np.column_stack( + [ + np.maximum(_numeric(full, c), 0.0) + for c in SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS + ] + ).reshape(balances.shape) + + has_balance = balances > 0 + federal = has_balance & (federal_code == _FEDERAL_LOAN_YES) + loans_1_6 = balances.sum(axis=1) + gross_up = np.ones(n, dtype=np.float64) + positive_detail = loans_1_6 > 0 + gross_up[positive_detail] = ( + 1.0 + remaining[positive_detail] / loans_1_6[positive_detail] + ) + federal_balance = np.where(federal, balances, 0.0) * gross_up[:, None] + + holder = np.full(balances.shape, _HEAD, dtype=object) + for code, role in SCF_WHOSE_EDUCATION_HOLDER.items(): + holder[whose == code] = role + is_head_own = federal & (holder == _HEAD) + is_spouse_own = federal & (holder == _SPOUSE) + is_parent_plus = federal & (holder == _PARENT_PLUS) + + head_own = np.where(is_head_own, federal_balance, 0.0).sum(axis=1) + spouse_own = np.where(is_spouse_own, federal_balance, 0.0).sum(axis=1) + parent_plus = np.where(is_parent_plus, federal_balance, 0.0).sum(axis=1) + total = head_own + spouse_own + parent_plus + + def _share(numerator: np.ndarray) -> np.ndarray: + return np.divide( + numerator, + total, + out=np.zeros(n, dtype=np.float64), + where=total > 0, + ) + + new_loan = year >= _IBR_NEW_BORROWER_FIRST_YEAR + + def _new_borrower(holds: np.ndarray) -> np.ndarray: + return (holds.any(axis=1) & np.all(new_loan | ~holds, axis=1)).astype( + np.float64 + ) + + weight = np.where(borrowed > 0, borrowed, balances) + graduate = np.isin(program, list(_GRADUATE_PROGRAM_CODES)) + + def _graduate_share(holds: np.ndarray) -> np.ndarray: + denominator = np.where(holds, weight, 0.0).sum(axis=1) + numerator = np.where(holds & graduate, weight, 0.0).sum(axis=1) + return np.divide( + numerator, + denominator, + out=np.zeros(n, dtype=np.float64), + where=denominator > 0, + ) + + result = pd.DataFrame(index=full.index) + result[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total + result["head_federal_student_loan_balance"] = head_own + result["spouse_federal_student_loan_balance"] = spouse_own + result["federal_parent_plus_loan_balance"] = parent_plus + result["head_own_share"] = _share(head_own) + result["spouse_own_share"] = _share(spouse_own) + result["head_graduate_share"] = _graduate_share(is_head_own) + result["spouse_graduate_share"] = _graduate_share(is_spouse_own) + result["head_is_ibr_new_borrower"] = _new_borrower(is_head_own | is_parent_plus) + result["spouse_is_ibr_new_borrower"] = _new_borrower(is_spouse_own) + result["education_debt"] = loans_1_6 + remaining + result["federal_status_unrecorded_balance"] = np.where( + has_balance & (federal_code == 0), balances, 0.0 + ).sum(axis=1) + result["remaining_loans_without_detail"] = np.where(positive_detail, 0.0, remaining) + return result + + +def load_scf_2025_education_debt_donor( + summary_extract_path: str | Path, + full_extract_path: str | Path, + *, + expected_rows: int | None = SCF_2025_FULL_EXTRACT.rows, +) -> pd.DataFrame: + """Join the 2025 summary and full files and derive the donor table. + + The two files must join one-to-one on ``(y1, yy1)`` across all five + implicates, each must have ``expected_rows`` rows (``None`` skips that + check, for test fixtures), and the per-loan balances must reproduce the + summary extract's ``edn_inst``. Returns one row per (family, implicate) + with a positive weight: the predictors, the targets of + :func:`derive_scf_education_debt_targets`, the ``scf_weight`` donor weight + and the fit-table covariates (``racecl4``, ``income``, ``educ``, + ``spouse_age``, ``yy1``, ``implicate``). + """ + + summary_columns = ( + *dict.fromkeys( + ( + *_SUMMARY_EXTRA_COLUMNS, + "wgt", + "age", + "hhsex", + "racecl5", + "married", + "kids", + "wageinc", + "intdivinc", + "ssretinc", + ) + ), + ) + full_columns = ( + "y1", + "yy1", + _SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN, + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + *SCF_EDUCATION_LOAN_YEAR_COLUMNS, + *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + ) + summary = _read_stata_columns( + summary_extract_path, summary_columns, label="2025 summary extract" + ) + full = _read_stata_columns(full_extract_path, full_columns, label="2025 full file") + keys = ["y1", "yy1"] + for label, table in (("summary", summary), ("full", full)): + if expected_rows is not None and len(table) != int(expected_rows): + raise ValueError( + f"SCF 2025 {label} file has {len(table)} rows; expected " + f"{int(expected_rows)}." + ) + if table.duplicated(keys).any(): + raise ValueError( + f"SCF 2025 {label} file does not have one-to-one (y1, yy1) keys." + ) + keyed_summary = summary.reset_index(drop=True) + keyed_summary["_summary_order"] = np.arange(len(keyed_summary)) + joined = keyed_summary.merge( + full, + on=keys, + how="outer", + validate="one_to_one", + indicator=True, + sort=False, + ) + unmatched = int((joined["_merge"] != "both").sum()) + if unmatched: + raise ValueError( + "SCF 2025 summary/full files have unmatched (y1, yy1) implicates; " + f"a one-to-one join is required ({unmatched} unmatched row(s))." + ) + joined = joined.sort_values("_summary_order").reset_index(drop=True) + implicate = pd.to_numeric(joined["y1"], errors="coerce").astype("int64") % 10 + if sorted(implicate.unique().tolist()) != [1, 2, 3, 4, 5]: + raise ValueError("SCF 2025 rows must carry implicates 1-5 (y1 mod 10).") + + targets = derive_scf_education_debt_targets(joined) + edn_inst = _numeric(joined, "edn_inst") + mismatch = np.abs(edn_inst - targets["education_debt"].to_numpy()) > 1.0 + if mismatch.any(): + raise ValueError( + "SCF 2025 education-loan balances do not reproduce edn_inst on " + f"{int(mismatch.sum())} row(s); the per-loan definition has drifted." + ) + + predictors = _scf_summary_predictor_table(joined) + donor = predictors.reset_index(drop=True) + donor["household_income"] = _numeric(joined, "income") - _numeric(joined, "kginc") + for column in targets.columns: + donor[column] = targets[column].to_numpy(dtype=np.float64) + donor["racecl4"] = _numeric(joined, "racecl4") + donor["income"] = _numeric(joined, "income") + donor["educ"] = _numeric(joined, "educ") + donor["spouse_age"] = _replace_sentinels( + joined[_SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN] + ).to_numpy(dtype=np.float64) + donor["yy1"] = pd.to_numeric(joined["yy1"], errors="coerce").to_numpy() + donor["implicate"] = implicate.to_numpy() + return donor.loc[donor[_DONOR_WEIGHT_COLUMN] > 0].reset_index(drop=True) + + +# --------------------------------------------------------------------------- # +# Recipients # +# --------------------------------------------------------------------------- # +@dataclass(frozen=True) +class _Recipients: + """Household-ordered predictors and each household's head and spouse rows.""" + + features: pd.DataFrame + head_position: np.ndarray + spouse_position: np.ndarray # -1 where no spouse or partner is present + + +def _resolve_partner_positions( + person: pd.DataFrame, head_positions: np.ndarray +) -> np.ndarray: + """Positional row of each head's spouse (A_SPOUSE) or partner (PECOHAB).""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + lookup = { + (household_id, int(line_number)): position + for position, (household_id, line_number) in enumerate( + zip(household, line, strict=True) + ) + } + pointers = [pd.to_numeric(person["A_SPOUSE"], errors="coerce").fillna(0).to_numpy()] + if _OPTIONAL_PARTNER_COLUMN in person.columns: + pointers.append( + pd.to_numeric(person[_OPTIONAL_PARTNER_COLUMN], errors="coerce") + .fillna(0) + .to_numpy() + ) + result = np.full(len(head_positions), -1, dtype=np.int64) + for index, head in enumerate(head_positions): + for pointer in pointers: + target = int(pointer[head]) + if target <= 0: + continue + partner = lookup.get((household[head], target)) + if partner is not None and partner != head: + result[index] = partner + break + return result + + +def _recipients(frame: Frame) -> _Recipients: + person = frame.table("person") + household = frame.table("household") + missing = sorted(set(_REQUIRED_RECIPIENT_PERSON_COLUMNS) - set(person.columns)) + if missing: + raise ValueError( + "US SCF education-debt imputation requires recipient person " + f"column(s): {missing}." + ) + if "household_id" not in household.columns: + raise ValueError("US SCF education-debt imputation requires household_id.") + + mask = _scf_reference_person_mask(person) + head_rows = np.flatnonzero(mask) + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + head_by_household = pd.Series(head_rows, index=household_ids[head_rows]) + if head_by_household.index.duplicated().any(): + raise ValueError("SCF education-debt receiver selected two reference persons.") + order = household["household_id"].to_numpy() + aligned = head_by_household.reindex(order) + if aligned.isna().any(): + missing_ids = order[aligned.isna().to_numpy()][:5].tolist() + raise ValueError( + "SCF education-debt receiver household alignment failed: " + f"no reference person for household id(s) {missing_ids}." + ) + head_position = aligned.to_numpy(dtype=np.int64) + + per_person = pd.DataFrame(index=np.arange(len(person))) + per_person["age"] = ( + pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + ) + per_person["is_female"] = person["is_female"].astype(bool).astype(float).to_numpy() + per_person["cps_race"] = _recipient_cps_race(person) + per_person["is_married"] = ( + pd.to_numeric(person["A_MARITL"], errors="coerce") + .isin([1, 2]) + .astype(float) + .to_numpy() + ) + per_person["own_children_in_household"] = _own_children_in_household(person) + + income = pd.DataFrame( + { + "household_id": household_ids, + "employment_income": np.maximum( + pd.to_numeric(person["employment_income_before_lsr"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64), + 0.0, + ), + "interest_dividend_income": _sum_present( + person, _RECIPIENT_INTEREST_DIVIDEND_COLUMNS + ), + "social_security_pension_income": _sum_present( + person, _RECIPIENT_SS_PENSION_COLUMNS + ), + "household_income": _sum_present( + person, _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS + ), + } + ) + household_income = income.groupby("household_id", sort=False).sum() + features = per_person.iloc[head_position].reset_index(drop=True) + sums = household_income.reindex(order) + for column in household_income.columns: + features[column] = sums[column].to_numpy(dtype=np.float64) + features.index = household.index + return _Recipients( + features=features.loc[:, list(SCF_EDUCATION_DEBT_PREDICTORS)], + head_position=head_position, + spouse_position=_resolve_partner_positions(person, head_position), + ) + + +# --------------------------------------------------------------------------- # +# Imputation # +# --------------------------------------------------------------------------- # +def _stage_seed(seed: int, step: int) -> int: + """A model seed for one of the stage's two QRFs, independent of other stages.""" + + sequence = np.random.SeedSequence([int(seed), 1073, int(step)]) + return int(sequence.generate_state(1, dtype=np.uint32)[0]) + + +def _fit_frame(donor: pd.DataFrame, columns: Sequence[str]) -> pd.DataFrame: + missing = [column for column in columns if column not in donor.columns] + if missing: + raise ValueError(f"SCF education-debt donor missing column(s): {missing}.") + fit = donor.loc[:, list(columns)].copy() + for column in columns: + fit[column] = pd.to_numeric(fit[column], errors="coerce") + if not np.isfinite(fit[column].to_numpy(dtype=np.float64)).all(): + raise ValueError( + f"SCF education-debt donor column {column!r} contains nonfinite values." + ) + return fit + + +def draw_scf_education_debt_households( + features: pd.DataFrame, + donor: pd.DataFrame, + *, + seed: int, + predictors: Sequence[str] = SCF_EDUCATION_DEBT_PREDICTORS, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> pd.DataFrame: + """Draw household totals and their composition for recipient rows. + + ``features`` carries the predictor columns, one row per recipient + household (or held-out SCF family). Returns, on the same index, the drawn + total and the normalized composition: the head's and spouse's own + balances, the head's Parent PLUS balance, both graduate shares and both + new-borrower flags. Used by the build and by the SCF hold-out fit check. + """ + + from microcosm.fit import QRF + + predictors = list(predictors) + total_fit = _fit_frame( + donor, (*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET, _DONOR_WEIGHT_COLUMN) + ) + total_model = QRF(n_estimators=int(n_estimators), seed=_stage_seed(seed, 1)).fit( + total_fit, + predictors=predictors, + targets=[SCF_EDUCATION_DEBT_TOTAL_TARGET], + weights=_DONOR_WEIGHT_COLUMN, + ) + drawn_total = np.maximum( + np.asarray( + total_model.predict(features.loc[:, predictors], release_models=True)[ + SCF_EDUCATION_DEBT_TOTAL_TARGET + ], + dtype=np.float64, + ), + 0.0, + ) + drawn_total = np.round(drawn_total) + + result = pd.DataFrame(index=features.index) + result[SCF_EDUCATION_DEBT_TOTAL_TARGET] = drawn_total + for column in SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: + result[column] = 0.0 + borrowers = drawn_total > 0 + if borrowers.any(): + composition_predictors = [*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET] + borrower_donor = donor.loc[ + pd.to_numeric(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], errors="coerce") > 0 + ] + composition_fit = _fit_frame( + borrower_donor, + ( + *composition_predictors, + *SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + _DONOR_WEIGHT_COLUMN, + ), + ) + composition_model = QRF( + n_estimators=int(n_estimators), seed=_stage_seed(seed, 2) + ).fit( + composition_fit, + predictors=composition_predictors, + targets=list(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS), + weights=_DONOR_WEIGHT_COLUMN, + ) + recipients = features.loc[borrowers, predictors].copy() + recipients[SCF_EDUCATION_DEBT_TOTAL_TARGET] = drawn_total[borrowers] + drawn = composition_model.predict(recipients, release_models=True) + for column in SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: + result.loc[borrowers, column] = np.asarray(drawn[column], dtype=np.float64) + + head_share = result["head_own_share"].clip(0.0, 1.0).to_numpy() + spouse_share = result["spouse_own_share"].clip(0.0, 1.0).to_numpy() + own = head_share + spouse_share + scale = np.where(own > 1.0, 1.0 / np.where(own > 0, own, 1.0), 1.0) + head_share = head_share * scale + spouse_share = spouse_share * scale + head_own = np.round(drawn_total * head_share) + # Whole dollars that never exceed the drawn total once rounded. + spouse_own = np.minimum( + np.round(drawn_total * spouse_share), drawn_total - head_own + ) + parent_plus = np.maximum(drawn_total - head_own - spouse_own, 0.0) + result["head_federal_student_loan_balance"] = head_own + result["spouse_federal_student_loan_balance"] = spouse_own + result["federal_parent_plus_loan_balance"] = parent_plus + for prefix, own_balance in (("head", head_own), ("spouse", spouse_own)): + graduate = result[f"{prefix}_graduate_share"].clip(0.0, 1.0).to_numpy() + result[f"{prefix}_graduate_share"] = np.where(own_balance > 0, graduate, 0.0) + head_holds = (head_own + parent_plus) > 0 + result["head_is_ibr_new_borrower"] = ( + head_holds & (result["head_is_ibr_new_borrower"].to_numpy() >= 0.5) + ).astype(np.float64) + result["spouse_is_ibr_new_borrower"] = ( + (spouse_own > 0) & (result["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5) + ).astype(np.float64) + result["head_own_share"] = head_share + result["spouse_own_share"] = spouse_share + return result + + +def impute_us_scf_education_debt( + frame: Frame, + donor: pd.DataFrame, + *, + seed: int, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> pd.DataFrame: + """Impute the four person columns onto a US frame. + + Returns a DataFrame indexed like the person table with + :data:`US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS`; only reference persons and + their spouses or partners can be nonzero. + """ + + recipients = _recipients(frame) + drawn = draw_scf_education_debt_households( + recipients.features, + donor, + seed=seed, + n_estimators=n_estimators, + ) + person = frame.table("person") + n_person = len(person) + own = np.zeros(n_person, dtype=np.float64) + parent_plus = np.zeros(n_person, dtype=np.float64) + graduate = np.zeros(n_person, dtype=np.float64) + new_borrower = np.zeros(n_person, dtype=bool) + + head = recipients.head_position + spouse = recipients.spouse_position + has_spouse = spouse >= 0 + head_own = drawn["head_federal_student_loan_balance"].to_numpy() + spouse_own = drawn["spouse_federal_student_loan_balance"].to_numpy() + head_graduate = drawn["head_graduate_share"].to_numpy() + spouse_graduate = drawn["spouse_graduate_share"].to_numpy() + head_new = drawn["head_is_ibr_new_borrower"].to_numpy() >= 0.5 + spouse_new = drawn["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5 + + # Without a spouse or partner present, the reference person holds the + # spouse's draw too: a balance-weighted graduate share, and new-borrower + # status only when both holders' loans are new. + moved = ~has_spouse & (spouse_own > 0) + combined_own = np.where(moved, head_own + spouse_own, head_own) + combined_graduate = np.where( + moved, + np.divide( + head_own * head_graduate + spouse_own * spouse_graduate, + combined_own, + out=np.zeros_like(combined_own), + where=combined_own > 0, + ), + head_graduate, + ) + head_holds_any = ( + head_own + drawn["federal_parent_plus_loan_balance"].to_numpy() + ) > 0 + combined_new = np.where( + moved, + np.where(head_holds_any, head_new, True) & spouse_new, + head_new, + ) + own[head] = combined_own + parent_plus[head] = drawn["federal_parent_plus_loan_balance"].to_numpy() + graduate[head] = combined_graduate + new_borrower[head] = combined_new + spouse_rows = spouse[has_spouse] + own[spouse_rows] = spouse_own[has_spouse] + graduate[spouse_rows] = spouse_graduate[has_spouse] + new_borrower[spouse_rows] = spouse_new[has_spouse] + + holds_any = (own + parent_plus) > 0 + result = pd.DataFrame(index=person.index) + result["federal_student_loan_balance"] = own + result["federal_parent_plus_loan_balance"] = parent_plus + result["is_income_based_repayment_new_borrower"] = new_borrower & holds_any + result["federal_student_loan_graduate_share"] = np.where(own > 0, graduate, 0.0) + return result + + +def _surface_has_signal(person: pd.DataFrame) -> bool: + if not all(column in person for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS): + return False + for column in US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS: + values = pd.to_numeric(person[column], errors="coerce") + if values.isna().any() or values.nunique() < 2 or not (values > 0).any(): + return False + return True + + +def with_us_scf_education_debt_inputs( + frame: Frame, + *, + seed: int, + time_period: int, + scf_education_debt_donor: pd.DataFrame, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> Frame: + """Write the four education-debt person inputs; idempotent on a signal.""" + + del time_period # The draw depends on the seed only; kept for the stage API. + if frame.schema != US_SCHEMA: + raise ValueError("US SCF education-debt inputs require the US schema.") + us_scf_education_debt_stage_spec() + person = frame.table("person") + if _surface_has_signal(person): + return frame + imputed = impute_us_scf_education_debt( + frame, + scf_education_debt_donor, + seed=int(seed), + n_estimators=int(n_estimators), + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS: + tables["person"][column] = imputed[column].to_numpy() + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +# --------------------------------------------------------------------------- # +# Stage declaration, diagnostics and gate # +# --------------------------------------------------------------------------- # +def us_scf_education_debt_stage_spec() -> SourceStageSpec: + """Load the ``scf_wealth`` declaration and check its 2025 education-debt part.""" + + manifest = load_source_manifest( + files("microcosm.build.us").joinpath("source_stages.json") + ) + spec = manifest.stage_map()[_SCF_STAGE_NAME] + missing = sorted(set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) - set(spec.outputs)) + if missing: + raise ValueError( + f"{_SCF_STAGE_NAME!r} manifest stage does not declare education-debt " + f"output(s) {missing}." + ) + for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT): + declared = [ + artifact + for artifact in spec.artifacts + if artifact.get("locator") == public_file.url + ] + if len(declared) != 1: + raise ValueError( + f"{_SCF_STAGE_NAME!r} manifest must declare {public_file.url} once." + ) + artifact = declared[0] + expected = { + "vintage": SCF_EDUCATION_DEBT_VINTAGE, + "member": public_file.member, + "sha256": public_file.zip_sha256, + "member_sha256": public_file.member_sha256, + } + drift = { + key: (artifact.get(key), value) + for key, value in expected.items() + if artifact.get(key) != value + } + if drift: + raise ValueError( + f"{_SCF_STAGE_NAME!r} SCF {SCF_EDUCATION_DEBT_VINTAGE} artifact pin " + f"drift for {public_file.member}: {drift}." + ) + fits = [ + operation + for operation in spec.operations + if operation.kind == "fit_weighted_qrf" + and operation.parameters.get("donor") == "scf_2025_education_debt" + ] + if len(fits) != 1 or tuple(fits[0].parameters.get("predictors", ())) != ( + SCF_EDUCATION_DEBT_PREDICTORS + ): + raise ValueError( + f"{_SCF_STAGE_NAME!r} education-debt QRF predictors have drifted." + ) + return spec + + +def _weighted_quantile(values: np.ndarray, weights: np.ndarray, q: float) -> float: + """First value whose cumulative weight reaches ``q`` of the total.""" + + if len(values) == 0 or float(weights.sum()) <= 0: + return float("nan") + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, q * cumulative[-1], side="left")) + return float(values[order][min(index, len(values) - 1)]) + + +def _band_shares( + values: np.ndarray, + weights: np.ndarray, + bands: Sequence[tuple[str, float, float]], + *, + inclusive_upper: bool, +) -> dict[str, float]: + total = float(weights.sum()) + shares: dict[str, float] = {} + for label, low, high in bands: + if inclusive_upper: + inside = (values >= low) & (values <= high) + else: + inside = (values >= low) & (values < high) + shares[label] = float(weights[inside].sum()) / total if total > 0 else 0.0 + return shares + + +def _income_quintiles(income: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Weighted income quintile (1-5) of each household.""" + + order = np.argsort(income, kind="mergesort") + cumulative = np.cumsum(weights[order]) + total = cumulative[-1] if len(cumulative) else 0.0 + quintile = np.empty(len(income), dtype=np.int64) + if total <= 0: + quintile[:] = 1 + return quintile + # Midpoint rule: a household's rank is the cumulative weight at its centre. + midpoint = (cumulative - weights[order] / 2.0) / total + quintile[order] = np.minimum((midpoint * 5).astype(np.int64) + 1, 5) + return quintile + + +def education_debt_fit_table( + *, + household_debt: np.ndarray, + household_weight: np.ndarray, + head_age: np.ndarray, + race_group: np.ndarray, + income_quintile: np.ndarray, + borrower_balance: np.ndarray, + borrower_age: np.ndarray, + borrower_weight: np.ndarray, + parent_plus_total: float, + graduate_numerator: float, + graduate_denominator: float, + new_borrower_weight: float, +) -> dict[str, object]: + """The microcosm#1073 fit table from household and borrower arrays. + + Household statistics follow the Fed's SCF tables (share of households with + federal education debt, mean and median among them, by the age of the + reference person, race and income quintile). Borrower statistics follow + Federal Student Aid's portfolio tables: people holding a federal balance + (own plus Parent PLUS), by balance band and borrower age band. + """ + + household_debt = np.asarray(household_debt, dtype=np.float64) + household_weight = np.asarray(household_weight, dtype=np.float64) + borrower = household_debt > 0 + total_weight = float(household_weight.sum()) + + def _group(mask: np.ndarray) -> dict[str, float]: + weight = household_weight[mask] + debt = household_debt[mask] + holders = debt > 0 + group_weight = float(weight.sum()) + holder_weight = float(weight[holders].sum()) + return { + "household_weight": group_weight, + "share_with_federal_education_debt": ( + holder_weight / group_weight if group_weight > 0 else float("nan") + ), + "mean_among_borrowers": ( + float(np.average(debt[holders], weights=weight[holders])) + if holder_weight > 0 + else float("nan") + ), + "median_among_borrowers": _weighted_quantile( + debt[holders], weight[holders], 0.5 + ), + } + + head_age = np.asarray(head_age, dtype=np.float64) + race_group = np.asarray(race_group) + income_quintile = np.asarray(income_quintile) + borrower_balance = np.asarray(borrower_balance, dtype=np.float64) + borrower_age = np.asarray(borrower_age, dtype=np.float64) + borrower_weight = np.asarray(borrower_weight, dtype=np.float64) + federal_total = float(np.dot(household_debt, household_weight)) + return { + "households": { + **_group(np.ones(len(household_debt), dtype=bool)), + "total_household_weight": total_weight, + "federal_education_debt_total": federal_total, + "parent_plus_share_of_total": ( + parent_plus_total / federal_total if federal_total > 0 else float("nan") + ), + }, + "household_balance_bands": _band_shares( + household_debt[borrower], + household_weight[borrower], + SCF_EDUCATION_DEBT_BALANCE_BANDS, + inclusive_upper=False, + ), + "by_head_age_band": { + label: _group((head_age >= low) & (head_age <= high)) + for label, low, high in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS + }, + "by_race": { + label: _group(race_group == code) for code, label in _RACECL4_LABELS.items() + }, + "by_income_quintile": { + str(quintile): _group(income_quintile == quintile) + for quintile in range(1, 6) + }, + "borrowers": { + "borrower_weight": float(borrower_weight.sum()), + "mean_balance": ( + float(np.average(borrower_balance, weights=borrower_weight)) + if float(borrower_weight.sum()) > 0 + else float("nan") + ), + "median_balance": _weighted_quantile( + borrower_balance, borrower_weight, 0.5 + ), + "balance_bands": _band_shares( + borrower_balance, + borrower_weight, + SCF_EDUCATION_DEBT_BALANCE_BANDS, + inclusive_upper=False, + ), + "age_bands": _band_shares( + np.floor(borrower_age), + borrower_weight, + SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + inclusive_upper=True, + ), + "graduate_share_of_own_original_principal": ( + graduate_numerator / graduate_denominator + if graduate_denominator > 0 + else float("nan") + ), + "ibr_new_borrower_share": ( + new_borrower_weight / float(borrower_weight.sum()) + if float(borrower_weight.sum()) > 0 + else float("nan") + ), + }, + } + + +def scf_education_debt_donor_fit_table( + donor: pd.DataFrame, + *, + imputed: pd.DataFrame | None = None, +) -> dict[str, object]: + """Fit table of the SCF donor itself, or of draws made for its rows. + + ``imputed`` (from :func:`draw_scf_education_debt_households`, on the + donor's index) replaces the observed targets, so a hold-out draw and the + observed SCF are tabulated identically. Implicates are pooled with + ``wgt`` (each implicate carries one fifth of the population), as the Fed + pools them for medians. + """ + + source = donor if imputed is None else imputed + weight = donor[_DONOR_WEIGHT_COLUMN].to_numpy(dtype=np.float64) + head_own = source["head_federal_student_loan_balance"].to_numpy(dtype=np.float64) + spouse_own = source["spouse_federal_student_loan_balance"].to_numpy( + dtype=np.float64 + ) + parent_plus = source["federal_parent_plus_loan_balance"].to_numpy(dtype=np.float64) + head_balance = head_own + parent_plus + head_holds = head_balance > 0 + spouse_holds = spouse_own > 0 + head_age = donor["age"].to_numpy(dtype=np.float64) + spouse_age = donor["spouse_age"].to_numpy(dtype=np.float64) + head_graduate = source["head_graduate_share"].to_numpy(dtype=np.float64) + spouse_graduate = source["spouse_graduate_share"].to_numpy(dtype=np.float64) + head_new = source["head_is_ibr_new_borrower"].to_numpy(dtype=np.float64) >= 0.5 + spouse_new = source["spouse_is_ibr_new_borrower"].to_numpy(dtype=np.float64) >= 0.5 + income = donor["household_income"].to_numpy(dtype=np.float64) + return education_debt_fit_table( + household_debt=head_own + spouse_own + parent_plus, + household_weight=weight, + head_age=head_age, + race_group=donor["racecl4"].to_numpy(), + income_quintile=_income_quintiles(income, weight), + borrower_balance=np.concatenate( + [head_balance[head_holds], spouse_own[spouse_holds]] + ), + borrower_age=np.concatenate([head_age[head_holds], spouse_age[spouse_holds]]), + borrower_weight=np.concatenate([weight[head_holds], weight[spouse_holds]]), + parent_plus_total=float(np.dot(parent_plus, weight)), + graduate_numerator=float( + np.dot(head_own * head_graduate + spouse_own * spouse_graduate, weight) + ), + graduate_denominator=float(np.dot(head_own + spouse_own, weight)), + new_borrower_weight=float( + weight[head_holds & head_new].sum() + + weight[spouse_holds & spouse_new].sum() + ), + ) + + +def _racecl4_from_cps(person: pd.DataFrame) -> np.ndarray: + """SCF ``racecl4`` grouping of CPS race and Hispanic origin.""" + + race = pd.to_numeric(person["PRDTRACE"], errors="coerce").fillna(0).to_numpy() + hispanic = pd.to_numeric(person["PRDTHSP"], errors="coerce").fillna(0).to_numpy() + group = np.full(len(person), 4, dtype=np.int64) + group[race == 1] = 1 + group[race == 2] = 2 + group[hispanic > 0] = 3 + return group + + +def us_scf_education_debt_fit_table(frame: Frame) -> dict[str, object]: + """The microcosm#1073 fit table of an imputed US frame, at its weights. + + Households are grouped by the SCF-style reference person's age and + ``racecl4``-equivalent race, and by weighted quintiles of the household + income the imputation conditions on (income quintiles of the frame's own + households, like the SCF's). + """ + + person = frame.table("person") + recipients = _recipients(frame) + household_weight = np.asarray( + frame.weights_for("household").values, dtype=np.float64 + ) + own = ( + pd.to_numeric(person["federal_student_loan_balance"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + parent_plus = ( + pd.to_numeric(person["federal_parent_plus_loan_balance"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + graduate = ( + pd.to_numeric(person["federal_student_loan_graduate_share"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + new_borrower = ( + person["is_income_based_repayment_new_borrower"].astype(bool).to_numpy() + ) + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + order = frame.table("household")["household_id"].to_numpy() + weight_by_household = pd.Series(household_weight, index=order) + person_weight = weight_by_household.reindex(household_ids).to_numpy( + dtype=np.float64 + ) + balance = own + parent_plus + household_debt = ( + pd.Series(balance, index=household_ids) + .groupby(level=0, sort=False) + .sum() + .reindex(order) + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + holds = balance > 0 + age = pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + head = recipients.head_position + return education_debt_fit_table( + household_debt=household_debt, + household_weight=household_weight, + head_age=age[head], + race_group=_racecl4_from_cps(person)[head], + income_quintile=_income_quintiles( + recipients.features["household_income"].to_numpy(dtype=np.float64), + household_weight, + ), + borrower_balance=balance[holds], + borrower_age=age[holds], + borrower_weight=person_weight[holds], + parent_plus_total=float(np.dot(parent_plus, person_weight)), + graduate_numerator=float(np.dot(own * graduate, person_weight)), + graduate_denominator=float(np.dot(own, person_weight)), + new_borrower_weight=float(person_weight[holds & new_borrower].sum()), + ) + + +def us_scf_education_debt_summary(frame: Frame) -> dict[str, object]: + """Weighted signal and consistency diagnostics for the four columns.""" + + person = frame.table("person") + present = [c for c in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS if c in person.columns] + summary: dict[str, object] = { + "present_columns": present, + "household_income_columns_present": [ + c for c in _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS if c in person.columns + ], + "borrower_household_share_band": list(_BORROWER_HOUSEHOLD_SHARE_BAND), + } + if len(present) != len(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS): + return summary + balances = { + column: pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64 + ) + for column in US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS + } + graduate = pd.to_numeric( + person["federal_student_loan_graduate_share"], errors="coerce" + ).to_numpy(dtype=np.float64) + new_borrower = person["is_income_based_repayment_new_borrower"] + holds = ( + balances["federal_student_loan_balance"] + + balances["federal_parent_plus_loan_balance"] + ) > 0 + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + order = frame.table("household")["household_id"].to_numpy() + household_weight = np.asarray( + frame.weights_for("household").values, dtype=np.float64 + ) + household_holds = ( + pd.Series(holds, index=household_ids) + .groupby(level=0, sort=False) + .any() + .reindex(order) + .fillna(False) + .to_numpy(dtype=bool) + ) + total_weight = float(household_weight.sum()) + summary.update( + { + "nonfinite_counts": { + column: int((~np.isfinite(values)).sum()) + for column, values in balances.items() + }, + "negative_counts": { + column: int((values < 0).sum()) for column, values in balances.items() + }, + "unique_counts": { + column: int(pd.Series(values).nunique()) + for column, values in balances.items() + }, + "graduate_share_out_of_range": int( + ((graduate < 0) | (graduate > 1) | ~np.isfinite(graduate)).sum() + ), + "graduate_share_without_own_balance": int( + ((graduate > 0) & (balances["federal_student_loan_balance"] <= 0)).sum() + ), + "new_borrower_dtype_is_bool": bool( + pd.api.types.is_bool_dtype(new_borrower) + ), + "new_borrower_without_balance": int( + (new_borrower.astype(bool).to_numpy() & ~holds).sum() + ), + "borrower_household_share": ( + float(household_weight[household_holds].sum()) / total_weight + if total_weight > 0 + else 0.0 + ), + } + ) + return summary + + +def us_scf_education_debt_signal_gate(frame: Frame) -> GateResult: + """Require finite, nonnegative, nonconstant and coherent education debt.""" + + summary = us_scf_education_debt_summary(frame) + missing = sorted( + set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) - set(summary["present_columns"]) + ) + if missing: + return GateResult( + name="scf_education_debt_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details=summary, + ) + failures: list[str] = [] + for column, count in summary["nonfinite_counts"].items(): + if count: + failures.append(f"{column}: {count} nonfinite value(s).") + for column, count in summary["negative_counts"].items(): + if count: + failures.append(f"{column}: {count} negative value(s).") + for column, count in summary["unique_counts"].items(): + if count < 2: + failures.append(f"{column}: constant column carries no signal.") + if summary["graduate_share_out_of_range"]: + failures.append( + "federal_student_loan_graduate_share: " + f"{summary['graduate_share_out_of_range']} value(s) outside [0, 1]." + ) + if summary["graduate_share_without_own_balance"]: + failures.append( + "federal_student_loan_graduate_share is positive for " + f"{summary['graduate_share_without_own_balance']} person(s) without " + "a federal_student_loan_balance." + ) + if not summary["new_borrower_dtype_is_bool"]: + failures.append("is_income_based_repayment_new_borrower is not boolean.") + if summary["new_borrower_without_balance"]: + failures.append( + "is_income_based_repayment_new_borrower is true for " + f"{summary['new_borrower_without_balance']} person(s) without a " + "federal balance." + ) + share = float(summary["borrower_household_share"]) + low, high = _BORROWER_HOUSEHOLD_SHARE_BAND + if not low <= share <= high: + failures.append( + f"federal education-debt household share {share:.3f} outside " + f"plausibility band [{low}, {high}]." + ) + return GateResult( + name="scf_education_debt_signal", + passed=not failures, + failures=tuple(failures), + details=summary, + ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py index ccec9ee148..4fa9ac6022 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py @@ -31,7 +31,7 @@ ROOT = _TEST_PATHS.repository US_ROOT = ROOT / "packages/microcosm-build/src/microcosm/build/us" CANONICAL_LEGACY_PROJECTION_SHA256 = ( - "05cb099ee8bd0234ae94586b76dc20ab8a5bd0735c00d4e82e67f912e0b33823" + "a086632ea6fed35b4d8010f141f4d317aa5ab43055cafad44ca2cae35e2cebca" ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py index b025205ae5..e9545c0a4a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py @@ -143,7 +143,7 @@ def test_ir_retains_all_typed_resolution_products(compiled_us: CompiledSpecIR) - assert len(inventory["columns"]) == 176 assert len(inventory["artifacts"]) == 84 assert len(inventory["scopes"]) == 7 - assert len(inventory["references"]) == 334 + assert len(inventory["references"]) == 336 assert "engine_abi_lock" in compiled_us.generated_authorities assert "records" in compiled_us.vintage_authorities assert set(compiled_us.surfaces) == { diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index a780c4814b..c2e864b912 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf", + "sha256": "c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" @@ -168,7 +168,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: sources = build_sources() vintages = build_vintages() records = vintages["records"] - assert len(records) == 16 + assert len(records) == 17 by_id = {record["id"]: record for record in records} assert set(by_id) == { "acs_2022", @@ -184,6 +184,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: "puma_2020", "release_us_2024", "scf_2022", + "scf_2025", "sipp_2023", "target_2024", "tax_2015", @@ -204,7 +205,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: } ) resolved_wire = thaw_json(resolved.vintage_authorities) - assert len(resolved_wire["records"]) == 16 + assert len(resolved_wire["records"]) == 17 assert ( resolved_wire["records"]["policyengine_us_surface"]["value"] == (ENGINE_LOCK["engine"]["version"]) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 9adfe409b7..5f7cb68360 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -2077,6 +2077,108 @@ def test_builder_exact_k_parser_enforces_charter_and_explicit_seed(capsys) -> No assert parsed.no_staging is True +def test_scf_education_debt_stage_is_opt_in_and_follows_auto_loans() -> None: + """microcosm#1073: the 2025 SCF stage runs only when asked, after auto loans. + + The pinned engine does not define the four federal student loan inputs + yet, so a default release must not fetch the 2025 SCF or carry them. + """ + + builder = _load_builder_module() + base_argv = ["--ledger-facts", "facts.jsonl", "--out", "release"] + parsed = builder._parse_args(base_argv) + assert parsed.scf_education_debt is False + assert parsed.scf_2025_summary_extract is None + assert parsed.scf_2025_full_extract is None + + parsed = builder._parse_args( + [ + *base_argv, + "--scf-education-debt", + "--scf-2025-summary-extract", + "rscfp2025.dta", + "--scf-2025-full-extract", + "p25i6.dta", + ] + ) + assert parsed.scf_education_debt is True + assert parsed.scf_2025_summary_extract == Path("rscfp2025.dta") + assert parsed.scf_2025_full_extract == Path("p25i6.dta") + + source = inspect.getsource(builder._main) + gate = source.index("if args.scf_education_debt:") + assert source.index("with_us_scf_auto_loan_inputs(") < gate + assert gate < source.index("_with_scf_education_debt_stage(") + assert source.index("join_acs_release_predictors(") < gate + + +def test_scf_education_debt_stage_runs_its_gate_and_writes_its_fit( + tmp_path, monkeypatch +) -> None: + builder = _load_builder_module() + calls: list[str] = [] + frame = object() + + monkeypatch.setattr( + builder, + "load_scf_2025_education_debt_donor", + lambda summary, full: calls.append(f"load:{summary}:{full}") or "donor", + ) + + def fake_with_inputs(base_frame, *, seed, time_period, scf_education_debt_donor): + assert base_frame is frame + assert scf_education_debt_donor == "donor" + calls.append(f"impute:{seed}:{time_period}") + return base_frame + + monkeypatch.setattr(builder, "with_us_scf_education_debt_inputs", fake_with_inputs) + monkeypatch.setattr( + builder, + "us_scf_education_debt_signal_gate", + lambda base_frame: builder.GateResult( + name="scf_education_debt_signal", passed=True + ), + ) + monkeypatch.setattr( + builder, + "us_scf_education_debt_fit_table", + lambda base_frame: {"households": {"share_with_federal_education_debt": 0.2}}, + ) + args = SimpleNamespace( + seed=7, + scf_2025_summary_extract=Path("rscfp2025.dta"), + scf_2025_full_extract=Path("p25i6.dta"), + ) + fit_path = tmp_path / "artifacts" / "us_scf_education_debt_fit.json" + assert ( + builder._with_scf_education_debt_stage( + frame, args=args, telemetry=None, fit_path=fit_path + ) + is frame + ) + assert calls == [ + f"load:{Path('rscfp2025.dta')}:{Path('p25i6.dta')}", + f"impute:7:{builder.PERIOD}", + ] + payload = json.loads(fit_path.read_text(encoding="utf-8")) + assert payload["weights"] == "pre_calibration_household_weights" + assert payload["fit"]["households"]["share_with_federal_education_debt"] == 0.2 + + monkeypatch.setattr( + builder, + "us_scf_education_debt_signal_gate", + lambda base_frame: builder.GateResult( + name="scf_education_debt_signal", + passed=False, + failures=("constant column",), + ), + ) + with pytest.raises(RuntimeError, match="SCF education-debt signal failed"): + builder._with_scf_education_debt_stage( + frame, args=args, telemetry=None, fit_path=None + ) + + def test_builder_gate_failed_base_pool_override_is_explicit(capsys) -> None: builder = _load_builder_module() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py new file mode 100644 index 0000000000..0e2e111bed --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py @@ -0,0 +1,671 @@ +"""SCF 2025 federal student loan stage tests (microcosm#1073). + +Federal education debt reads the 2025 SCF: per-loan federal flags, whose +education each loan paid for, the year borrowed and the program financed. The +stage writes four person inputs that policyengine-us#9721 and #9724 add. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import zipfile + +import numpy as np +import pandas as pd +import pytest + +import microcosm.build.us_runtime.scf_education_debt as education_debt +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + SCF_EDUCATION_LOAN_YEAR_COLUMNS, + US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, + ScfPublicFile, + derive_scf_education_debt_targets, + draw_scf_education_debt_households, + education_debt_fit_table, + fetch_scf_public_file, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, + us_scf_education_debt_stage_spec, + us_scf_education_debt_summary, + with_us_scf_education_debt_inputs, +) +from microcosm.build.us_runtime.scf_wealth import SCF_WEALTH_PREDICTORS +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +TIME_PERIOD = 2024 +_DONOR_WEIGHT_COLUMN = "scf_weight" +_LOAN_ITEMS = { + "balance": SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + "federal": SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + "whose": SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + "year": SCF_EDUCATION_LOAN_YEAR_COLUMNS, + "program": SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + "borrowed": SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, +} + + +def _family(*loans: dict[str, float], remaining: float = 0.0) -> dict[str, float]: + """One full-file row: up to six loans given as item dictionaries.""" + + row = {column: 0.0 for columns in _LOAN_ITEMS.values() for column in columns} + row[SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN] = remaining + for index, loan in enumerate(loans): + for item, value in loan.items(): + row[_LOAN_ITEMS[item][index]] = value + return row + + +def _loan( + balance: float, + *, + federal: int = 1, + whose: int = 1, + year: int = 2018, + program: int = 12, + borrowed: float | None = None, +) -> dict[str, float]: + return { + "balance": balance, + "federal": federal, + "whose": whose, + "year": year, + "program": program, + "borrowed": balance if borrowed is None else borrowed, + } + + +def _derive(*families: dict[str, float]) -> pd.DataFrame: + return derive_scf_education_debt_targets(pd.DataFrame.from_records(families)) + + +# --------------------------------------------------------------------------- # +# Donor targets # +# --------------------------------------------------------------------------- # +def test_only_federal_loans_count_and_holders_follow_whose_education() -> None: + row = _derive( + _family( + _loan(10_000, whose=1, year=2016, program=12), + _loan(20_000, whose=2, year=2010, program=13, borrowed=25_000), + _loan(5_000, whose=3, year=2019, program=12), + _loan(7_000, federal=5, whose=1), + _loan(3_000, federal=0, whose=1), + ) + ).iloc[0] + + assert row["head_federal_student_loan_balance"] == 10_000 + assert row["spouse_federal_student_loan_balance"] == 20_000 + assert row["federal_parent_plus_loan_balance"] == 5_000 + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 35_000 + assert row["head_own_share"] == pytest.approx(10_000 / 35_000) + assert row["spouse_own_share"] == pytest.approx(20_000 / 35_000) + # The head holds the 2016 own loan and the 2019 Parent PLUS loan. + assert row["head_is_ibr_new_borrower"] == 1.0 + assert row["spouse_is_ibr_new_borrower"] == 0.0 + assert row["head_graduate_share"] == 0.0 + assert row["spouse_graduate_share"] == 1.0 + assert row["education_debt"] == 45_000 + assert row["federal_status_unrecorded_balance"] == 3_000 + + +def test_remaining_loans_take_the_family_federal_share_and_holder_split() -> None: + row = _derive( + _family( + _loan(6_000, whose=1), + _loan(4_000, federal=5, whose=1), + _loan(2_000, whose=3), + remaining=6_000, + ) + ).iloc[0] + + # Loans 1-6 total 12,000, so each is scaled by 1 + 6,000 / 12,000. + assert row["head_federal_student_loan_balance"] == pytest.approx(9_000) + assert row["federal_parent_plus_loan_balance"] == pytest.approx(3_000) + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == pytest.approx(12_000) + assert row["education_debt"] == pytest.approx(18_000) + assert row["remaining_loans_without_detail"] == 0.0 + + +def test_remaining_loans_without_detail_are_dropped_and_reported() -> None: + row = _derive(_family(remaining=4_000)).iloc[0] + + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 0.0 + assert row["remaining_loans_without_detail"] == 4_000 + assert row["head_own_share"] == 0.0 + assert row["head_is_ibr_new_borrower"] == 0.0 + + +@pytest.mark.parametrize("code", [6, 7, -7, 0, 4]) +def test_multiple_unrelated_other_and_unknown_codes_go_to_the_head(code) -> None: + row = _derive(_family(_loan(8_000, whose=code))).iloc[0] + + assert row["head_federal_student_loan_balance"] == 8_000 + assert row["spouse_federal_student_loan_balance"] == 0.0 + assert row["federal_parent_plus_loan_balance"] == 0.0 + assert row["head_own_share"] == 1.0 + + +def test_graduate_share_weights_by_amount_borrowed_with_balance_fallback() -> None: + weighted = _derive( + _family( + _loan(5_000, program=14, borrowed=30_000), + _loan(9_000, program=11, borrowed=10_000), + ) + ).iloc[0] + fallback = _derive( + _family( + _loan(5_000, program=15, borrowed=0.0), + _loan(15_000, program=16, borrowed=0.0), + ) + ).iloc[0] + parent_plus_only = _derive(_family(_loan(5_000, whose=3, program=13))).iloc[0] + + assert weighted["head_graduate_share"] == pytest.approx(0.75) + assert fallback["head_graduate_share"] == pytest.approx(0.25) + # A Parent PLUS loan is not the parent's own loan. + assert parent_plus_only["head_graduate_share"] == 0.0 + + +def test_new_borrower_status_counts_2014_as_new_and_needs_every_loan() -> None: + rows = _derive( + _family(_loan(1_000, year=2014)), + _family(_loan(1_000, year=2013)), + _family(_loan(1_000, year=2020), _loan(1_000, year=2005, whose=3)), + _family(_loan(1_000, year=2020), _loan(1_000, year=2005, federal=5)), + ) + + assert rows["head_is_ibr_new_borrower"].tolist() == [1.0, 0.0, 0.0, 1.0] + + +def test_derivation_refuses_a_missing_loan_item() -> None: + frame = pd.DataFrame.from_records([_family(_loan(1_000))]).drop(columns="x9338") + with pytest.raises(ValueError, match="x9338"): + derive_scf_education_debt_targets(frame) + + +# --------------------------------------------------------------------------- # +# Loader # +# --------------------------------------------------------------------------- # +def _summary_and_full(n_families: int = 6) -> tuple[pd.DataFrame, pd.DataFrame]: + summary_rows: list[dict[str, float]] = [] + full_rows: list[dict[str, float]] = [] + for family in range(1, n_families + 1): + loans = [_loan(1_000.0 * family, whose=1 + family % 3)] if family % 2 else [] + for implicate in range(1, 6): + y1 = family * 10 + implicate + full = {"y1": float(y1), "yy1": float(family), "x19": 40.0 + family} + full.update(_family(*loans)) + full_rows.append(full) + debt = sum(loan["balance"] for loan in loans) + summary_rows.append( + { + "y1": float(y1), + "yy1": float(family), + "wgt": 1_000.0 + family, + "age": 25.0 + family, + "hhsex": 1.0 + family % 2, + "racecl5": float(family % 5 + 1), + "racecl4": float(family % 4 + 1), + "married": float(family % 2), + "kids": float(family % 3), + "wageinc": 30_000.0 + family, + "intdivinc": 100.0 * family, + "ssretinc": 0.0, + "income": 40_000.0 + 1_000.0 * family, + "kginc": 500.0 * family, + "edn_inst": debt, + "educ": 12.0, + } + ) + return pd.DataFrame(summary_rows), pd.DataFrame(full_rows).iloc[::-1] + + +def _write(tmp_path, summary: pd.DataFrame, full: pd.DataFrame): + summary_path = tmp_path / "rscfp2025.dta" + full_path = tmp_path / "p25i6.dta" + summary.to_stata(summary_path, write_index=False) + full.to_stata(full_path, write_index=False) + return summary_path, full_path + + +def test_loader_joins_both_files_and_derives_targets_and_predictors(tmp_path) -> None: + summary, full = _summary_and_full() + donor = load_scf_2025_education_debt_donor( + *_write(tmp_path, summary, full), expected_rows=None + ) + + assert len(donor) == len(summary) + assert set(SCF_EDUCATION_DEBT_PREDICTORS) <= set(donor.columns) + assert set(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS) <= set(donor.columns) + first = donor.iloc[0] + assert first["household_income"] == pytest.approx(41_000.0 - 500.0) + assert first["spouse_age"] == 41.0 + assert sorted(donor["implicate"].unique()) == [1, 2, 3, 4, 5] + family_three = donor.loc[donor["yy1"] == 3].iloc[0] + assert family_three[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 3_000.0 + assert family_three["head_federal_student_loan_balance"] == 3_000.0 + + +def test_loader_refuses_drifted_education_debt_and_row_counts(tmp_path) -> None: + summary, full = _summary_and_full() + with pytest.raises(ValueError, match="expected 30"): + load_scf_2025_education_debt_donor( + *_write(tmp_path, summary.iloc[:-5], full.iloc[:-5]), expected_rows=30 + ) + drifted = summary.copy() + drifted.loc[0, "edn_inst"] = 99_999.0 + with pytest.raises(ValueError, match="edn_inst"): + load_scf_2025_education_debt_donor( + *_write(tmp_path, drifted, full), expected_rows=None + ) + + +def test_loader_refuses_an_unmatched_implicate(tmp_path) -> None: + summary, full = _summary_and_full() + full = full.copy() + full.iloc[0, full.columns.get_loc("y1")] = 999.0 + with pytest.raises(ValueError, match="unmatched"): + load_scf_2025_education_debt_donor( + *_write(tmp_path, summary, full), expected_rows=None + ) + + +# --------------------------------------------------------------------------- # +# Declaration and source files # +# --------------------------------------------------------------------------- # +def test_stage_declares_the_person_outputs_and_pinned_2025_files() -> None: + spec = us_scf_education_debt_stage_spec() + + assert set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) <= set(spec.outputs) + locators = {artifact.get("locator"): artifact for artifact in spec.artifacts} + for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT): + artifact = locators[public_file.url] + assert artifact["vintage"] == "2025" + assert artifact["sha256"] == public_file.zip_sha256 + assert artifact["member_sha256"] == public_file.member_sha256 + assert artifact["expected_rows"] == public_file.rows + # Net worth and the other wealth outputs stay on the 2022 files. + assert spec.artifacts[0]["vintage"] == "2022" + assert spec.artifacts[1]["vintage"] == "2022" + assert spec.artifacts[2]["member"] == "pu2023.csv" + + +def _zip_with(member: str, payload: bytes) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(member, payload) + return buffer.getvalue() + + +class _Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_fetch_verifies_both_pins_and_reuses_a_verified_cache( + tmp_path, monkeypatch +) -> None: + member = b"stata bytes" + archive = _zip_with("p25i6.dta", member) + pinned = ScfPublicFile( + url="https://example.invalid/scf2025s.zip", + member="p25i6.dta", + zip_sha256=hashlib.sha256(archive).hexdigest(), + member_sha256=hashlib.sha256(member).hexdigest(), + rows=1, + ) + fetched: list[str] = [] + + def fake_urlopen(request, timeout): + fetched.append(request.full_url) + return _Response(archive) + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + path = fetch_scf_public_file(pinned, tmp_path) + assert path.read_bytes() == member + assert fetch_scf_public_file(pinned, tmp_path) == path + assert fetched == [pinned.url] + + wrong_zip = ScfPublicFile( + url=pinned.url, + member=pinned.member, + zip_sha256="0" * 64, + member_sha256=pinned.member_sha256, + rows=1, + ) + with pytest.raises(ValueError, match="sha-256"): + fetch_scf_public_file(wrong_zip, tmp_path / "other") + + +# --------------------------------------------------------------------------- # +# Imputation # +# --------------------------------------------------------------------------- # +def _ready_donor(n: int = 600) -> pd.DataFrame: + rng = np.random.default_rng(1073) + donor = pd.DataFrame({name: rng.normal(size=n) for name in SCF_WEALTH_PREDICTORS}) + donor["age"] = rng.integers(20, 85, n).astype(np.float64) + donor["is_female"] = rng.integers(0, 2, n).astype(np.float64) + donor["cps_race"] = rng.integers(1, 8, n).astype(np.float64) + donor["is_married"] = rng.integers(0, 2, n).astype(np.float64) + donor["own_children_in_household"] = rng.integers(0, 3, n).astype(np.float64) + donor["employment_income"] = rng.gamma(2.0, 25_000.0, n) + donor["interest_dividend_income"] = rng.gamma(1.0, 800.0, n) + donor["social_security_pension_income"] = rng.gamma(1.0, 5_000.0, n) + donor["household_income"] = donor["employment_income"] + rng.gamma(1.0, 5_000.0, n) + borrower = (rng.random(n) < 0.2) & (donor["age"] < 65) + total = np.where(borrower, rng.gamma(1.5, 25_000.0, n).round(), 0.0) + holder = rng.choice(3, n, p=[0.6, 0.25, 0.15]) + head_share = np.where(borrower, (holder == 0).astype(float), 0.0) + spouse_share = np.where(borrower, (holder == 1).astype(float), 0.0) + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total + donor["head_own_share"] = head_share + donor["spouse_own_share"] = spouse_share + donor["head_graduate_share"] = np.where( + head_share > 0, rng.choice([0.0, 0.5, 1.0], n), 0.0 + ) + donor["spouse_graduate_share"] = np.where( + spouse_share > 0, rng.choice([0.0, 1.0], n), 0.0 + ) + donor["head_is_ibr_new_borrower"] = np.where( + borrower & (holder != 1), (rng.random(n) < 0.5).astype(float), 0.0 + ) + donor["spouse_is_ibr_new_borrower"] = np.where( + spouse_share > 0, (rng.random(n) < 0.5).astype(float), 0.0 + ) + donor[_DONOR_WEIGHT_COLUMN] = rng.uniform(500.0, 2_000.0, n) + return donor + + +def _person_rows(n_households: int = 150) -> pd.DataFrame: + """Couples (A_SPOUSE pointers) and single adults with a child.""" + + rng = np.random.default_rng(97) + records: list[dict[str, object]] = [] + person_id = 1 + for household_id in range(1, n_households + 1): + couple = household_id % 3 != 0 + members = ( + [(1, 2, 1, 0), (2, 1, 1, 1), (3, 0, 7, 0)] + if couple + else [(1, 0, 7, 0), (2, 0, 7, 1)] + ) + for line, spouse, marital, female in members: + child = line == 3 or (not couple and line == 2) + records.append( + { + "person_id": person_id, + "person_household_id": household_id, + "PH_SEQ": household_id, + "A_LINENO": line, + "A_SPOUSE": spouse, + "age": float( + rng.integers(5, 17) if child else rng.integers(24, 60) + ), + "is_female": bool(female), + "PRDTRACE": int(rng.integers(1, 5)), + "PRDTHSP": int(rng.integers(0, 2)), + "A_MARITL": marital, + "PEPAR1": 1 if child else -1, + "PEPAR2": -1, + "employment_income_before_lsr": 0.0 + if child + else float(rng.gamma(2.0, 20_000.0)), + "taxable_interest_income": float(rng.gamma(1.0, 300.0)), + } + ) + person_id += 1 + return pd.DataFrame.from_records(records) + + +def _us_frame(person: pd.DataFrame) -> Frame: + person = person.copy() + household_ids = np.sort(person["person_household_id"].unique()) + n_person = len(person) + person["person_tax_unit_id"] = person["person_household_id"] + 1_000 + person["person_spm_unit_id"] = person["person_household_id"] + 2_000 + person["person_family_id"] = person["person_household_id"] + 3_000 + person["person_marital_unit_id"] = np.arange(n_person) + 4_000 + tables = { + "person": person, + "household": pd.DataFrame({"household_id": household_ids}), + "tax_unit": pd.DataFrame({"tax_unit_id": household_ids + 1_000}), + "spm_unit": pd.DataFrame({"spm_unit_id": household_ids + 2_000}), + "family": pd.DataFrame({"family_id": household_ids + 3_000}), + "marital_unit": pd.DataFrame({"marital_unit_id": np.arange(n_person) + 4_000}), + } + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + values=np.full(len(household_ids), 1_000.0), + kind=WeightKind.DESIGN, + ) + }, + ) + + +def test_household_draws_are_seeded_coherent_and_nonnegative() -> None: + donor = _ready_donor() + features = donor.loc[:, list(SCF_EDUCATION_DEBT_PREDICTORS)].iloc[:200] + first = draw_scf_education_debt_households(features, donor, seed=3, n_estimators=15) + second = draw_scf_education_debt_households( + features, donor, seed=3, n_estimators=15 + ) + pd.testing.assert_frame_equal(first, second) + + total = first[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy() + parts = ( + first["head_federal_student_loan_balance"] + + first["spouse_federal_student_loan_balance"] + + first["federal_parent_plus_loan_balance"] + ).to_numpy() + assert (total >= 0).all() and (total > 0).any() + np.testing.assert_array_equal(parts, total) + assert (first["head_own_share"] + first["spouse_own_share"] <= 1 + 1e-9).all() + borrowers = total > 0 + assert not first.loc[~borrowers, list(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS)].any( + axis=None + ) + + +def test_imputation_reaches_only_reference_persons_and_their_spouses() -> None: + frame = _us_frame(_person_rows()) + imputed = impute_us_scf_education_debt( + frame, _ready_donor(), seed=5, n_estimators=15 + ) + person = frame.table("person") + + assert list(imputed.columns) == list(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) + adult = person["age"] >= 18 + balance = ( + imputed["federal_student_loan_balance"] + + imputed["federal_parent_plus_loan_balance"] + ) + assert (balance >= 0).all() and (balance > 0).any() + assert (balance[~adult] == 0).all() + # Parent PLUS sits on the reference person (line 1 in these fixtures). + assert ( + imputed.loc[person["A_LINENO"] != 1, "federal_parent_plus_loan_balance"] == 0 + ).all() + own = imputed["federal_student_loan_balance"] + assert (imputed.loc[own <= 0, "federal_student_loan_graduate_share"] == 0).all() + assert imputed["federal_student_loan_graduate_share"].between(0, 1).all() + assert imputed["is_income_based_repayment_new_borrower"].dtype == bool + assert not ( + imputed["is_income_based_repayment_new_borrower"] & (balance <= 0) + ).any() + single_adults = person["A_MARITL"].eq(7) & adult + assert (person.loc[single_adults, "A_LINENO"] == 1).all() + + +def test_a_spouse_draw_without_a_spouse_moves_to_the_reference_person() -> None: + frame = _us_frame(_person_rows(30)) + donor = _ready_donor() + donor["head_own_share"] = np.where(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 0, 0) + donor["spouse_own_share"] = np.where( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 1.0, 0.0 + ) + donor["head_graduate_share"] = 0.0 + donor["head_is_ibr_new_borrower"] = 0.0 + imputed = impute_us_scf_education_debt(frame, donor, seed=11, n_estimators=10) + person = frame.table("person") + + single = person["person_household_id"] % 3 == 0 + reference = person["A_LINENO"] == 1 + couple_spouse = ~single & (person["A_LINENO"] == 2) + own = imputed["federal_student_loan_balance"] + assert (own[single & ~reference] == 0).all() + assert (own[single & reference] > 0).any() + assert (own[~single & reference] == 0).all() + assert (own[couple_spouse] > 0).any() + assert (imputed["federal_parent_plus_loan_balance"] == 0).all() + + +def test_with_inputs_is_idempotent_and_passes_its_gate() -> None: + frame = _us_frame(_person_rows()) + out = with_us_scf_education_debt_inputs( + frame, + seed=1, + time_period=TIME_PERIOD, + scf_education_debt_donor=_ready_donor(), + n_estimators=15, + ) + again = with_us_scf_education_debt_inputs( + out, + seed=2, + time_period=TIME_PERIOD, + scf_education_debt_donor=_ready_donor(), + n_estimators=15, + ) + assert again is out + gate = us_scf_education_debt_signal_gate(out) + assert gate.passed, gate.failures + summary = us_scf_education_debt_summary(out) + assert summary["household_income_columns_present"] == [ + "employment_income_before_lsr", + "taxable_interest_income", + ] + fit = us_scf_education_debt_fit_table(out) + assert 0 < fit["households"]["share_with_federal_education_debt"] < 1 + assert set(fit["borrowers"]["balance_bands"]) == { + label for label, _, _ in education_debt.SCF_EDUCATION_DEBT_BALANCE_BANDS + } + + +def test_gate_refuses_missing_negative_and_incoherent_columns() -> None: + frame = _us_frame(_person_rows()) + missing = us_scf_education_debt_signal_gate(frame) + assert not missing.passed + assert "person columns missing" in missing.failures[0] + + out = with_us_scf_education_debt_inputs( + frame, + seed=1, + time_period=TIME_PERIOD, + scf_education_debt_donor=_ready_donor(), + n_estimators=15, + ) + tables = {entity: out.table(entity).copy() for entity in out.entities} + person = tables["person"] + person.loc[person.index[0], "federal_student_loan_balance"] = -1.0 + no_balance = person["federal_student_loan_balance"] <= 0 + person.loc[no_balance, "federal_student_loan_graduate_share"] = 0.5 + person["is_income_based_repayment_new_borrower"] = True + broken = Frame( + tables, + out.schema, + {entity: out.weights_for(entity) for entity in out.weighted_entities}, + ) + failures = " ".join(us_scf_education_debt_signal_gate(broken).failures) + assert "negative" in failures + assert "without a federal_student_loan_balance" in failures + assert "without a federal balance" in failures + + +# --------------------------------------------------------------------------- # +# Fit table # +# --------------------------------------------------------------------------- # +def test_fit_table_matches_hand_computed_shares_medians_and_bands() -> None: + table = education_debt_fit_table( + household_debt=np.array([0.0, 4_000.0, 30_000.0, 250_000.0]), + household_weight=np.array([2.0, 1.0, 1.0, 1.0]), + head_age=np.array([70.0, 24.0, 30.0, 40.0]), + race_group=np.array([1, 1, 2, 3]), + income_quintile=np.array([1, 2, 3, 5]), + borrower_balance=np.array([4_000.0, 30_000.0, 250_000.0]), + borrower_age=np.array([24.0, 30.0, 40.0]), + borrower_weight=np.array([1.0, 1.0, 2.0]), + parent_plus_total=25_000.0, + graduate_numerator=10.0, + graduate_denominator=40.0, + new_borrower_weight=1.0, + ) + + households = table["households"] + assert households["share_with_federal_education_debt"] == pytest.approx(3 / 5) + assert households["mean_among_borrowers"] == pytest.approx(284_000 / 3) + assert households["median_among_borrowers"] == 30_000.0 + assert households["parent_plus_share_of_total"] == pytest.approx(25_000 / 284_000) + assert table["household_balance_bands"]["under_5k"] == pytest.approx(1 / 3) + assert table["household_balance_bands"]["200k_plus"] == pytest.approx(1 / 3) + assert ( + table["by_head_age_band"]["62_and_older"]["share_with_federal_education_debt"] + == 0.0 + ) + assert table["by_race"]["white_non_hispanic"][ + "share_with_federal_education_debt" + ] == pytest.approx(1 / 3) + borrowers = table["borrowers"] + assert borrowers["median_balance"] == 30_000.0 + assert borrowers["age_bands"]["24_and_younger"] == pytest.approx(0.25) + assert borrowers["age_bands"]["35_to_49"] == pytest.approx(0.5) + assert borrowers["graduate_share_of_own_original_principal"] == 0.25 + assert borrowers["ibr_new_borrower_share"] == pytest.approx(0.25) + + +def test_donor_fit_table_tabulates_observed_and_imputed_rows_alike() -> None: + donor = _ready_donor(200) + donor["head_federal_student_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["head_own_share"] + ) + donor["spouse_federal_student_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["spouse_own_share"] + ) + donor["federal_parent_plus_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] + - donor["head_federal_student_loan_balance"] + - donor["spouse_federal_student_loan_balance"] + ) + donor["racecl4"] = (np.arange(len(donor)) % 4 + 1).astype(float) + donor["spouse_age"] = donor["age"] + observed = scf_education_debt_donor_fit_table(donor) + same = scf_education_debt_donor_fit_table(donor, imputed=donor) + assert json.dumps(observed, sort_keys=True) == json.dumps(same, sort_keys=True) + expected_share = float( + donor.loc[ + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, _DONOR_WEIGHT_COLUMN + ].sum() + / donor[_DONOR_WEIGHT_COLUMN].sum() + ) + assert observed["households"]["share_with_federal_education_debt"] == ( + pytest.approx(expected_share) + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 284149bdf1..9efca30be4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -308,6 +308,9 @@ "release_input_coverage.py", "release_target_parity.py", "scf_auto_loans.py", + # SCF 2025 federal student loan inputs (microcosm#1073); a donor-model + # stage shaped like scf_auto_loans. + "scf_education_debt.py", "scf_wealth.py", "sipp_financial_assets.py", "sipp_tips.py", diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py b/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py index af5b7bbf64..e94d102ee3 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py @@ -13,7 +13,8 @@ def test_every_register_entry_is_a_named_producer_column(): base-spine tag of every US entity, the pooled-source provenance columns, microunit's two tax-unit construction columns, the PUF capital-gains tail provenance columns, the ACS-native acs_ amounts and acs_pums's puma_geoid - alias.""" + alias, plus the two pending engine-input families: the parent ids and the + federal student loan inputs.""" support = { column(entity) @@ -44,9 +45,10 @@ def test_every_register_entry_is_a_named_producer_column(): | _ACS_NATIVE_AMOUNTS | {"puma_geoid"} | set(US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS) + | set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) ) assert len(_ACS_NATIVE_AMOUNTS) == 6 - assert len(US_STORED_NON_VARIABLE_COLUMNS) == 30 + 6 + 6 + 6 + 1 + 2 + assert len(US_STORED_NON_VARIABLE_COLUMNS) == 30 + 6 + 6 + 6 + 1 + 2 + 4 def test_every_register_reason_names_where_its_column_comes_from(): @@ -79,6 +81,10 @@ def test_every_register_reason_names_where_its_column_comes_from(): column: "US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS" for column in US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS }, + **{ + column: "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS" + for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS + }, } for column, owner in expected_owner.items(): assert owner in US_STORED_NON_VARIABLE_COLUMNS[column], column diff --git a/packages/microcosm-data/src/microcosm/data/stored_inputs.py b/packages/microcosm-data/src/microcosm/data/stored_inputs.py index f1ad592d76..8415050aef 100644 --- a/packages/microcosm-data/src/microcosm/data/stored_inputs.py +++ b/packages/microcosm-data/src/microcosm/data/stored_inputs.py @@ -269,7 +269,10 @@ #: (microcosm#884). No examined file stores them yet, so they rest on their #: live producer, ``eligibility_inputs.US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS``, #: which ``test_us_stored_input_register.py`` binds, and leave the register -#: once the pinned engine defines them. +#: once the pinned engine defines them. The four federal student loan inputs +#: (microcosm#1073) likewise postdate them: their producer is +#: ``scf_education_debt.US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS``, and they leave +#: the register once the pinned engine carries policyengine-us#9721 and #9724. #: #: Two model-named non-variable columns that the published default, its #: receipt child and the ACS local-area release store are deliberately absent, @@ -428,6 +431,42 @@ ("parent_2_id", "PEPAR2"), ) }, + **{ + column: ( + f"{detail}, imputed from the 2025 Survey of Consumer Finances by " + "microcosm.build.us_runtime.scf_education_debt " + "(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS; microcosm#1073), and " + "written only when the fiscal-refresh build runs with " + "--scf-education-debt. The certified engine does not define it " + f"yet, so it is ignored there; policyengine-us#{pull} adds it as " + "an input. Remove this entry when the pinned engine defines the " + "variable." + ) + for column, detail, pull in ( + ( + "federal_student_loan_balance", + "The person's own federal student loan balance", + 9721, + ), + ( + "federal_parent_plus_loan_balance", + "The person's federal Parent PLUS loan balance", + 9721, + ), + ( + "is_income_based_repayment_new_borrower", + "Whether every federal loan the person holds was borrowed in " + "2014 or later (IBR new-borrower status)", + 9724, + ), + ( + "federal_student_loan_graduate_share", + "The graduate or professional share of the original principal " + "of the person's own federal student loans", + 9724, + ), + ) + }, } ) diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index ef4bc8868b..10c3280084 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/test_support/microcosm_build/us_stored_input_register.py b/test_support/microcosm_build/us_stored_input_register.py index 91c008846b..f191a253e4 100644 --- a/test_support/microcosm_build/us_stored_input_register.py +++ b/test_support/microcosm_build/us_stored_input_register.py @@ -31,6 +31,9 @@ US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS, ) from microcosm.build.us_runtime.operator_boundary import _ACS_NATIVE_INPUT_CONTRACTS +from microcosm.build.us_runtime.scf_education_debt import ( + US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, +) from microcosm.build.us_runtime.support_provenance import ( spine_source_id_column, support_channel_column, diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 6b64ce7059..195c3ad8be 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -79,6 +79,8 @@ CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, ORG_2024_DONOR_CONTENT_SHA256, + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256, SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES, SIPP_2023_HEAD_START_DONOR_SHA256, @@ -109,6 +111,7 @@ fetch_org_2024_donor, fetch_scf_2022_full_extract, fetch_scf_2022_summary_extract, + fetch_scf_public_file, fetch_sipp_2023_financial_asset_donor, fetch_sipp_2023_tip_donor, hard_target_package_aliases, @@ -117,6 +120,7 @@ load_org_2024_donor, load_scf_2022_auto_loan_donor, load_scf_2022_financial_asset_donor, + load_scf_2025_education_debt_donor, load_sipp_2023_financial_asset_donor, load_sipp_2023_head_start_donor, load_sipp_2023_ssi_disability_donor, @@ -162,6 +166,8 @@ us_retirement_distributions_signal_gate, us_salt_refund_income_signal_gate, us_scf_auto_loans_signal_gate, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, us_scf_wealth_signal_gate, us_sipp_head_start_signal_gate, us_sipp_tips_signal_gate, @@ -200,6 +206,7 @@ with_us_retirement_contribution_inputs, with_us_retirement_distribution_inputs, with_us_scf_auto_loan_inputs, + with_us_scf_education_debt_inputs, with_us_scf_wealth_inputs, with_us_sipp_head_start_input, with_us_sipp_tip_inputs, @@ -1593,6 +1600,35 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "interest. When omitted, scf2022s.zip is fetched and cached." ), ) + parser.add_argument( + "--scf-education-debt", + action="store_true", + help=( + "Impute the four federal student loan person inputs " + "(federal_student_loan_balance, federal_parent_plus_loan_balance, " + "is_income_based_repayment_new_borrower, " + "federal_student_loan_graduate_share) from the 2025 SCF " + "(microcosm#1073). Off by default until the pinned policyengine-us " + "defines them (policyengine-us#9721, #9724); the certified engine " + "ignores them until then." + ), + ) + parser.add_argument( + "--scf-2025-summary-extract", + type=Path, + help=( + "Optional local rscfp2025.dta for --scf-education-debt. When " + "omitted, the sha-pinned scfp2025s.zip is fetched and cached." + ), + ) + parser.add_argument( + "--scf-2025-full-extract", + type=Path, + help=( + "Optional local p25i6.dta for --scf-education-debt. When omitted, " + "the sha-pinned scf2025s.zip is fetched and cached." + ), + ) parser.add_argument( "--sipp-tip-donor", type=Path, @@ -12380,6 +12416,78 @@ def _check_committed_us_ledger_feed_pin( ) +def _with_scf_education_debt_stage( + base_frame: Frame, + *, + args: argparse.Namespace, + telemetry: Any, + fit_path: Path | None, +) -> Frame: + """Run the opt-in SCF 2025 education-debt stage and its signal gate.""" + + if telemetry is not None: + telemetry.stage( + "scf_education_debt_inputs", + message=( + "Imputing federal student loan balances, Parent PLUS balances, " + "IBR new-borrower status and graduate shares from the 2025 SCF." + ), + ) + summary_path = ( + Path(args.scf_2025_summary_extract) + if args.scf_2025_summary_extract is not None + else fetch_scf_public_file(SCF_2025_SUMMARY_EXTRACT) + ) + full_path = ( + Path(args.scf_2025_full_extract) + if args.scf_2025_full_extract is not None + else fetch_scf_public_file(SCF_2025_FULL_EXTRACT) + ) + donor = load_scf_2025_education_debt_donor(summary_path, full_path) + base_frame = with_us_scf_education_debt_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + scf_education_debt_donor=donor, + ) + gate = us_scf_education_debt_signal_gate(base_frame) + if not gate.passed: + if telemetry is not None: + telemetry.stage( + "scf_education_debt_gate", + status="failed", + message="SCF education-debt signal gate failed.", + failures=list(gate.failures), + force_upload=True, + ) + raise RuntimeError( + "Release gates failed: " + + "; ".join( + f"SCF education-debt signal failed: {failure}" + for failure in gate.failures + ) + ) + if fit_path is not None: + # Design-weight fit of the staged frame against the SCF; the receipt + # tool tabulates the calibrated export the same way (microcosm#1073). + fit_path.parent.mkdir(parents=True, exist_ok=True) + fit_path.write_text( + json.dumps( + { + "schema_version": 1, + "weights": "pre_calibration_household_weights", + "fit": us_scf_education_debt_fit_table(base_frame), + }, + indent=2, + sort_keys=True, + default=float, + ) + + "\n", + encoding="utf-8", + ) + return base_frame + + def _main(argv: Sequence[str] | None = None) -> int | None: global _ACTIVE_EMITTER, _ACTIVE_PROGRESS _ACTIVE_EMITTER = None @@ -13961,6 +14069,21 @@ def _main(argv: Sequence[str] | None = None) -> int | None: for failure in scf_auto_loan_gate.failures ) ) + if args.scf_education_debt: + # microcosm#1073: federal student loan balances from the 2025 SCF. The + # pinned engine does not define these inputs yet, so the stage is + # opt-in and its columns ride in the stored-input register until it + # does (microcosm.data.stored_inputs). + base_frame = _with_scf_education_debt_stage( + base_frame, + args=args, + telemetry=telemetry, + fit_path=( + artifact_root / "us_scf_education_debt_fit.json" + if dry_run is None + else None + ), + ) if telemetry is not None: telemetry.stage( "sipp_vehicle_inputs", diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 5dc45835db..89d025cebf 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/us_bundle_generation/core.py b/tools/us_bundle_generation/core.py index aa6448f100..38c38d9aa6 100644 --- a/tools/us_bundle_generation/core.py +++ b/tools/us_bundle_generation/core.py @@ -911,6 +911,16 @@ def build_vintages() -> dict[str, Any]: "field": "vintage", }, }, + { + "id": "scf_2025", + "kind": "survey_period_ref", + "authority_ref": { + "kind": "source_stage_artifact", + "stage": "scf_wealth", + "artifact_index": 3, + "field": "vintage", + }, + }, { "id": "sipp_2023", "kind": "survey_period_ref", @@ -958,6 +968,7 @@ def build_vintages() -> dict[str, Any]: "puma_2020": {"cd_119", "census_2020"}, "release_us_2024": {"target_2024"}, "scf_2022": {"target_2024"}, + "scf_2025": {"target_2024"}, "sipp_2023": {"target_2024"}, "target_2024": { "acs_2022", @@ -969,6 +980,7 @@ def build_vintages() -> dict[str, Any]: "policyengine_us_surface", "release_us_2024", "scf_2022", + "scf_2025", "sipp_2023", "tax_2015", }, From c0ccddad3eef41041ef274d0ac21a7e21715ad58 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Mon, 12 Oct 2026 14:25:36 -0400 Subject: [PATCH 02/17] Add SCF 2025 education-debt fit receipts (#1073) Weighted fit of the stage against SCF 2025: source checks against the Fed's Alternate Table 13 (19.92%, $25,500, matched exactly), a five-fold family hold-out with a predictor ablation, and the stage imputed onto the published default populace-us-2024-spm-20260915 at its weights. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-education-debt-receipts.md | 153 +++ .../receipts/1073-scf-education-debt-fit.json | 1023 +++++++++++++++++ 2 files changed, 1176 insertions(+) create mode 100644 experiments/1073-scf-education-debt-receipts.md create mode 100644 experiments/receipts/1073-scf-education-debt-fit.json diff --git a/experiments/1073-scf-education-debt-receipts.md b/experiments/1073-scf-education-debt-receipts.md new file mode 100644 index 0000000000..4abf2c176e --- /dev/null +++ b/experiments/1073-scf-education-debt-receipts.md @@ -0,0 +1,153 @@ +# microcosm#1073 receipts: SCF 2025 federal student loan balances + +Generated 2026-10-12 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`. Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. + +Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5). Household figures group by the reference person's age and race and by weighted quintiles of household income less capital gains, the income concept the imputation conditions on. Borrower figures count people with a federal balance (own plus Parent PLUS), in Federal Student Aid's balance and age bands; SCF borrowers are the reference person and spouse. Nothing here is a gate or a calibration target. + +## Source checks + +| Check | Loader | Fed Alternate Table 13 | +|---|---:|---:| +| Families with any education debt | 19.92% | 19.92% | +| Median education debt among them | $25,500 | $25,500 | + +| Share of education-loan balances | Value | +|---|---:| +| loans 1 6 of all education debt | 99.4% | +| x7179 remaining loans of all education debt | 0.6% | +| federal of loans 1 6 | 87.4% | +| non federal of loans 1 6 | 12.0% | +| federal status blank of loans 1 6 | 0.6% | + +| Federal balances on loans 1-6, by whose education | Share | +|---|---:| +| 1 respondent (reference person) | 54.8% | +| 2 spouse or partner | 31.4% | +| 3 child (incl. grandchild, other relative) | 13.8% | +| 6 multiple people | 0.0% | +| 7 unrelated person | 0.0% | +| -7 other | 0.0% | + +## Fit + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| Households with federal education debt | 18.2% | 17.3% | 14.4% | +| Mean among them | $41,052 | $39,267 | $44,483 | +| Median among them | $25,000 | $23,000 | $25,000 | +| Weighted federal education debt | $1.006T | $0.916T | $0.786T | +| Parent PLUS share of the total | 13.8% | 11.6% | 21.5% | +| Borrowers (people) | 27.2M | 24.7M | 18.5M | +| Mean balance per borrower | $36,934 | $37,030 | $42,554 | +| Median balance per borrower | $21,000 | $22,000 | $24,000 | +| IBR new borrowers among borrowers | 59.2% | 59.2% | 60.6% | +| Graduate share of own original principal | 47.7% | 46.2% | 53.6% | + +### Borrowers by FSA balance band + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| under_5k | 11.9% | 10.8% | 10.4% | +| 5k_to_10k | 15.5% | 15.2% | 16.7% | +| 10k_to_20k | 18.6% | 18.3% | 15.5% | +| 20k_to_40k | 22.3% | 24.0% | 23.0% | +| 40k_to_60k | 12.8% | 12.2% | 10.4% | +| 60k_to_80k | 8.1% | 9.4% | 9.6% | +| 80k_to_100k | 3.7% | 3.5% | 5.1% | +| 100k_to_200k | 4.9% | 4.8% | 6.4% | +| 200k_plus | 2.2% | 1.9% | 2.9% | + +### Borrowers by FSA age band + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| 24_and_younger | 5.7% | 6.1% | 3.8% | +| 25_to_34 | 33.1% | 33.3% | 26.6% | +| 35_to_49 | 38.1% | 38.4% | 39.8% | +| 50_to_61 | 16.2% | 15.8% | 21.1% | +| 62_and_older | 6.9% | 6.4% | 8.7% | + +### Households with federal education debt, by age of reference person + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| 24_and_younger | 22.2% | 24.4% | 22.7% | +| 25_to_34 | 36.2% | 34.7% | 26.7% | +| 35_to_49 | 27.2% | 27.2% | 22.3% | +| 50_to_61 | 16.7% | 14.5% | 14.2% | +| 62_and_older | 4.2% | 3.3% | 3.7% | + +Median among borrowers: + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| 24_and_younger | $10,000 | $11,000 | $6,000 | +| 25_to_34 | $24,900 | $24,600 | $20,200 | +| 35_to_49 | $25,000 | $22,000 | $25,670 | +| 50_to_61 | $28,000 | $28,000 | $29,000 | +| 62_and_older | $26,000 | $28,000 | $38,000 | + +### Households with federal education debt, by race (`racecl4`) + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| white_non_hispanic | 17.2% | 15.9% | 13.3% | +| black_non_hispanic | 24.3% | 22.5% | 22.8% | +| hispanic | 14.1% | 16.1% | 12.3% | +| other_or_multiple | 19.5% | 19.2% | 15.3% | + +Median among borrowers: + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| white_non_hispanic | $23,000 | $24,000 | $25,000 | +| black_non_hispanic | $30,000 | $24,600 | $28,000 | +| hispanic | $23,000 | $20,000 | $24,764 | +| other_or_multiple | $20,000 | $22,000 | $24,000 | + +### Households with federal education debt, by income quintile + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| 1 | 11.4% | 10.3% | 12.7% | +| 2 | 17.1% | 17.1% | 13.6% | +| 3 | 17.8% | 17.9% | 13.4% | +| 4 | 26.6% | 25.5% | 20.0% | +| 5 | 17.9% | 15.7% | 12.4% | + +Median among borrowers: + +| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | +|---|---:|---:|---:| +| 1 | $19,200 | $19,200 | $20,000 | +| 2 | $22,000 | $22,000 | $26,000 | +| 3 | $21,000 | $23,000 | $23,000 | +| 4 | $28,000 | $25,000 | $25,000 | +| 5 | $28,000 | $26,000 | $28,000 | + +## Predictor ablation (five-fold hold-out on SCF families) + +Each variant refits both QRFs. Errors are drawn minus observed; TVD is total variation distance between band distributions; MAE averages the absolute error of the share with debt across the groups. `educ` is the SCF reference person's education, which the CPS side cannot supply until microcosm#297, so that row shows what the predictor would add. + +| Variant | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | household_rank_correlation | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| shared_8_without_age | -0.250 | -1.737 | -4.000 | 0.022 | 4.812 | 0.648 | 0.672 | 0.028 | 0.107 | 0.154 | +| shared_8 | -0.532 | -4.092 | -8.000 | 0.041 | 1.193 | 1.078 | 0.876 | 0.037 | 0.016 | 0.173 | +| shared_8_plus_household_income (build) | -0.873 | -4.350 | -8.000 | 0.037 | 1.351 | 1.359 | 0.923 | 0.030 | 0.009 | 0.175 | +| plus_household_income_and_education | -1.474 | -5.567 | -4.816 | 0.040 | 1.321 | 2.067 | 1.474 | 0.050 | 0.023 | 0.208 | + +## Host fit + +Imputed with build seed 0 onto `populace-us-2024-spm-20260915` (57,240 households, sha256 `6496cc4393d4...`) at its calibrated household weights; the published weights were calibrated without these columns. Signal gate: passed. + +| Metric (imputed minus SCF 2025) | Value | +|---|---:| +| share_with_debt_error_pp | -3.742 | +| mean_among_borrowers_error_pct | 8.357 | +| median_among_borrowers_error_pct | 0.000 | +| household_balance_band_tvd | 0.062 | +| head_age_band_share_mae_pp | 3.554 | +| race_share_mae_pp | 2.834 | +| income_quintile_share_mae_pp | 4.252 | +| borrower_balance_band_tvd | 0.070 | +| borrower_age_band_tvd | 0.085 | diff --git a/experiments/receipts/1073-scf-education-debt-fit.json b/experiments/receipts/1073-scf-education-debt-fit.json new file mode 100644 index 0000000000..5a17f1b568 --- /dev/null +++ b/experiments/receipts/1073-scf-education-debt-fit.json @@ -0,0 +1,1023 @@ +{ + "schema_version": 1, + "issue": "PolicyEngine/microcosm#1073", + "generated_at": "2026-10-12", + "scf_files": { + "rscfp2025.dta": { + "url": "https://www.federalreserve.gov/econres/files/scfp2025s.zip", + "zip_sha256": "32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + "member_sha256": "454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd" + }, + "p25i6.dta": { + "url": "https://www.federalreserve.gov/econres/files/scf2025s.zip", + "zip_sha256": "c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + "member_sha256": "38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad" + } + }, + "source_checks": { + "fed_alternate_table_13": { + "source": "Federal Reserve, 2025 SCF Chartbook tables, scf2025_tables_public_real_historical.xlsx, sheet 'Table 13 25 Alt %s & median', education loans", + "share_of_families_with_education_debt_pct": 19.92, + "median_among_holders": 25500.0 + }, + "loader_share_of_families_with_education_debt_pct": 19.916506070519297, + "loader_median_among_holders": 25500.0, + "families_per_implicate": 4360, + "balance_shares": { + "loans_1_6_of_all_education_debt": 0.9943433718962694, + "x7179_remaining_loans_of_all_education_debt": 0.0056566281037305766, + "federal_of_loans_1_6": 0.8742168884171948, + "non_federal_of_loans_1_6": 0.11976788840960186, + "federal_status_blank_of_loans_1_6": 0.006015223173203283 + }, + "federal_balance_share_by_whose_education_code": { + "1 respondent (reference person)": 0.5478808107398941, + "2 spouse or partner": 0.3138757801591027, + "3 child (incl. grandchild, other relative)": 0.13824340910100308, + "6 multiple people": 0.0, + "7 unrelated person": 0.0, + "-7 other": 0.0 + }, + "whose_education_codes_outside_the_rule": [], + "federal_education_debt_after_x7179_rule": 1005887392198.396, + "remaining_loans_without_detail_dollars": 0.0 + }, + "scf_2025_observed": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.18154956812200357, + "mean_among_borrowers": 41052.27209260245, + "median_among_borrowers": 25000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 1005887392198.396, + "parent_plus_share_of_total": 0.13750075307389342 + }, + "household_balance_bands": { + "under_5k": 0.09639294713632403, + "5k_to_10k": 0.14003280730185305, + "10k_to_20k": 0.1763887243762958, + "20k_to_40k": 0.23167145676213327, + "40k_to_60k": 0.13228448002879964, + "60k_to_80k": 0.09356242722856564, + "80k_to_100k": 0.04781176224843039, + "100k_to_200k": 0.05847272768581226, + "200k_plus": 0.023382667231785843 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.22183897089190358, + "mean_among_borrowers": 23531.319738797185, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3615365625657711, + "mean_among_borrowers": 36363.72833498089, + "median_among_borrowers": 24900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.27186430754586627, + "mean_among_borrowers": 45297.2739464529, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.1666515695303606, + "mean_among_borrowers": 44901.11056030461, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.041870913769168856, + "mean_among_borrowers": 41544.318165395336, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.17240693757923897, + "mean_among_borrowers": 41150.07823437558, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.24343266231667623, + "mean_among_borrowers": 47375.69026752636, + "median_among_borrowers": 30000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.1409034533516125, + "mean_among_borrowers": 33382.63507234893, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.19450371139945485, + "mean_among_borrowers": 37451.352530441625, + "median_among_borrowers": 20000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.1144757325207629, + "mean_among_borrowers": 30747.429065421915, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.1705804998024808, + "mean_among_borrowers": 41222.638862750704, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17774016899539674, + "mean_among_borrowers": 38756.18543238133, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26595688145561025, + "mean_among_borrowers": 40379.14671664771, + "median_among_borrowers": 28000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.1790121945626286, + "mean_among_borrowers": 50757.93410007803, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 27234421.461229563, + "mean_balance": 36934.413812695064, + "median_balance": 21000.0, + "balance_bands": { + "under_5k": 0.11892493269111482, + "5k_to_10k": 0.15531044452506518, + "10k_to_20k": 0.18601428472545406, + "20k_to_40k": 0.22258528435724315, + "40k_to_60k": 0.1281197371592323, + "60k_to_80k": 0.08084824950055876, + "80k_to_100k": 0.03718912377212063, + "100k_to_200k": 0.04910146852901287, + "200k_plus": 0.021906474740198353 + }, + "age_bands": { + "24_and_younger": 0.057262304197073094, + "25_to_34": 0.33134066182556904, + "35_to_49": 0.3807800839788549, + "50_to_61": 0.1615926964748789, + "62_and_older": 0.06902425352362417 + }, + "graduate_share_of_own_original_principal": 0.47738241507384116, + "ibr_new_borrower_share": 0.5915143468879298 + } + }, + "holdout": { + "folds": 5, + "seed": 0, + "n_estimators": 100, + "production_variant": "shared_8_plus_household_income", + "variants": { + "shared_8_without_age": { + "predictors": [ + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17905210701848603, + "mean_among_borrowers": 40339.111086866, + "median_among_borrowers": 24000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 974816127786.9578, + "parent_plus_share_of_total": 0.12332712997930678 + }, + "household_balance_bands": { + "under_5k": 0.10118107773912553, + "5k_to_10k": 0.13505410339066057, + "10k_to_20k": 0.18318555601050834, + "20k_to_40k": 0.21964423310630496, + "40k_to_60k": 0.13583862553597464, + "60k_to_80k": 0.1008575608439786, + "80k_to_100k": 0.045400028448036055, + "100k_to_200k": 0.05696009741342364, + "200k_plus": 0.021878717511987597 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.2856494213931131, + "mean_among_borrowers": 33090.297708091784, + "median_among_borrowers": 16000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.2634230533073922, + "mean_among_borrowers": 39081.51087614914, + "median_among_borrowers": 25000.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2511547364155009, + "mean_among_borrowers": 41977.825875862174, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.20137196508720936, + "mean_among_borrowers": 39932.6451255711, + "median_among_borrowers": 23000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.06509257053181323, + "mean_among_borrowers": 42570.021847555945, + "median_among_borrowers": 28000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.16722261212813858, + "mean_among_borrowers": 39982.38559368015, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.236098978018357, + "mean_among_borrowers": 45093.93722051315, + "median_among_borrowers": 27000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.15009122535417432, + "mean_among_borrowers": 34558.186278806694, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1987172729932262, + "mean_among_borrowers": 39719.50100870802, + "median_among_borrowers": 20200.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.1250378708596092, + "mean_among_borrowers": 32495.496996203394, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.16848970888296705, + "mean_among_borrowers": 40638.90333292065, + "median_among_borrowers": 23000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17317139260393358, + "mean_among_borrowers": 38283.29263303069, + "median_among_borrowers": 23900.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26089759644033866, + "mean_among_borrowers": 43703.342247747096, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16768340855350833, + "mean_among_borrowers": 42776.28156983026, + "median_among_borrowers": 27000.0 + } + }, + "borrowers": { + "borrower_weight": 25995647.800317153, + "mean_balance": 37499.20506982191, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.12187825205948413, + "5k_to_10k": 0.1410220365125263, + "10k_to_20k": 0.1851654547328621, + "20k_to_40k": 0.21348131319748645, + "40k_to_60k": 0.13547790411800253, + "60k_to_80k": 0.09385336282040968, + "80k_to_100k": 0.04113173480343179, + "100k_to_200k": 0.0501832187776321, + "200k_plus": 0.017806722978165053 + }, + "age_bands": { + "24_and_younger": 0.07113074708615924, + "25_to_34": 0.24900499283093128, + "35_to_49": 0.3562472636589869, + "50_to_61": 0.21093410606503074, + "62_and_older": 0.11268289035889188 + }, + "graduate_share_of_own_original_principal": 0.4521386518636319, + "ibr_new_borrower_share": 0.5904131759381358 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.24974611035175343, + "mean_among_borrowers_error_pct": -1.7372022774470364, + "median_among_borrowers_error_pct": -4.0000000000000036, + "household_balance_band_tvd": 0.022434241359602005, + "head_age_band_share_mae_pp": 4.81151166418894, + "race_share_mae_pp": 0.6479835836438199, + "income_quintile_share_mae_pp": 0.6721955334843016, + "borrower_balance_band_tvd": 0.028340960926920844, + "borrower_age_band_tvd": 0.10686848931450572, + "household_rank_correlation": 0.15376545505926326 + } + }, + "shared_8": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17622772670671366, + "mean_among_borrowers": 39372.39907723004, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 936446699439.1069, + "parent_plus_share_of_total": 0.12480448512425421 + }, + "household_balance_bands": { + "under_5k": 0.09343877143258432, + "5k_to_10k": 0.16316715085533556, + "10k_to_20k": 0.1722357902778136, + "20k_to_40k": 0.22990980510936418, + "40k_to_60k": 0.12014043404307863, + "60k_to_80k": 0.1082920780850408, + "80k_to_100k": 0.03372552360552188, + "100k_to_200k": 0.0612588533574213, + "200k_plus": 0.017831593233839888 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.2529895671878583, + "mean_among_borrowers": 24415.889909464808, + "median_among_borrowers": 11143.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3600484792377984, + "mean_among_borrowers": 35943.04970065601, + "median_among_borrowers": 23244.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2683793916575207, + "mean_among_borrowers": 41209.59995046775, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.14913648907893393, + "mean_among_borrowers": 41823.4843446509, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03583557010066891, + "mean_among_borrowers": 51630.83962020747, + "median_among_borrowers": 36000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.1630534163575822, + "mean_among_borrowers": 38775.16233534788, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.2289947160752199, + "mean_among_borrowers": 47894.9650347352, + "median_among_borrowers": 27000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.1566171160511706, + "mean_among_borrowers": 33164.275274056155, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1981094869447479, + "mean_among_borrowers": 35811.82171400663, + "median_among_borrowers": 20911.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.12126660088067846, + "mean_among_borrowers": 32701.99311366745, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.1559905239732057, + "mean_among_borrowers": 37718.60645255383, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17955131715072067, + "mean_among_borrowers": 38482.880144490715, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26346013869266394, + "mean_among_borrowers": 40527.082922078465, + "median_among_borrowers": 23195.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16089043903093728, + "mean_among_borrowers": 45104.97225220377, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 25263299.640741862, + "mean_balance": 37067.473875380434, + "median_balance": 21634.0, + "balance_bands": { + "under_5k": 0.11215076141602473, + "5k_to_10k": 0.16261828276540408, + "10k_to_20k": 0.1743648469362625, + "20k_to_40k": 0.22983573580807584, + "40k_to_60k": 0.12284267042874972, + "60k_to_80k": 0.0949401843921611, + "80k_to_100k": 0.029332915740791533, + "100k_to_200k": 0.057418087395041346, + "200k_plus": 0.016496515117489015 + }, + "age_bands": { + "24_and_younger": 0.06351111547591387, + "25_to_34": 0.3387564753526982, + "35_to_49": 0.37092134292463996, + "50_to_61": 0.1558115646143939, + "62_and_older": 0.07099950163235395 + }, + "graduate_share_of_own_original_principal": 0.4607883219614359, + "ibr_new_borrower_share": 0.5830953867882087 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.5321841415289907, + "mean_among_borrowers_error_pct": -4.092034203571204, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.040650120081566585, + "head_age_band_share_mae_pp": 1.1934803926439914, + "race_share_mae_pp": 1.077772642699106, + "income_quintile_share_mae_pp": 0.8762098127830446, + "borrower_balance_band_tvd": 0.03696684344880254, + "borrower_age_band_tvd": 0.015639872914699815, + "household_rank_correlation": 0.17265294243545534 + } + }, + "shared_8_plus_household_income": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17281627661600854, + "mean_among_borrowers": 39266.70030095491, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 915853474784.7556, + "parent_plus_share_of_total": 0.11558498718060217 + }, + "household_balance_bands": { + "under_5k": 0.08974701999643789, + "5k_to_10k": 0.1480250881106031, + "10k_to_20k": 0.1777595352503208, + "20k_to_40k": 0.24623523553505594, + "40k_to_60k": 0.11977492174284053, + "60k_to_80k": 0.10688238472563866, + "80k_to_100k": 0.03949620364967014, + "100k_to_200k": 0.051414568653002084, + "200k_plus": 0.020665042336430926 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.24417812288361246, + "mean_among_borrowers": 29778.191701476153, + "median_among_borrowers": 11000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3474148475378485, + "mean_among_borrowers": 36843.74904984769, + "median_among_borrowers": 24600.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2717612000881406, + "mean_among_borrowers": 39300.240621441015, + "median_among_borrowers": 22000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.14495396516593925, + "mean_among_borrowers": 42167.25806737501, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03258898091103795, + "mean_among_borrowers": 51808.82295387341, + "median_among_borrowers": 28000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.15882220643031406, + "mean_among_borrowers": 38343.12358920671, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.224885334931214, + "mean_among_borrowers": 44936.30170394143, + "median_among_borrowers": 24600.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.16095386588360475, + "mean_among_borrowers": 34860.299992999004, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1923427751654244, + "mean_among_borrowers": 39117.56739436241, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.10260826272789049, + "mean_among_borrowers": 33872.567578817216, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.17100662350830068, + "mean_among_borrowers": 39205.32920511589, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.1785510426393907, + "mean_among_borrowers": 36610.58351254444, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.25540364615718963, + "mean_among_borrowers": 37762.92241632937, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.1565282419252913, + "mean_among_borrowers": 48351.71138032323, + "median_among_borrowers": 26000.0 + } + }, + "borrowers": { + "borrower_weight": 24732420.497163415, + "mean_balance": 37030.48291976904, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.10800838456724018, + "5k_to_10k": 0.15183570250699477, + "10k_to_20k": 0.18301823120692082, + "20k_to_40k": 0.2396488060359636, + "40k_to_60k": 0.1219331041101281, + "60k_to_80k": 0.09403649589832407, + "80k_to_100k": 0.03487135041034836, + "100k_to_200k": 0.04757724842740005, + "200k_plus": 0.01907067683668007 + }, + "age_bands": { + "24_and_younger": 0.06096826553348787, + "25_to_34": 0.3331677198859893, + "35_to_49": 0.38398055689037613, + "50_to_61": 0.15779746384936916, + "62_and_older": 0.06408599384077752 + }, + "graduate_share_of_own_original_principal": 0.4619857518712365, + "ibr_new_borrower_share": 0.5919421297398388 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343, + "household_rank_correlation": 0.17515687252147844 + } + }, + "plus_household_income_and_education": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income", + "educ" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.1668063934073779, + "mean_among_borrowers": 38767.059113449155, + "median_among_borrowers": 23796.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 872755299812.5964, + "parent_plus_share_of_total": 0.11953256305130716 + }, + "household_balance_bands": { + "under_5k": 0.08808431892789516, + "5k_to_10k": 0.13717658256161608, + "10k_to_20k": 0.18873503407584216, + "20k_to_40k": 0.24545468460672265, + "40k_to_60k": 0.12322636723840236, + "60k_to_80k": 0.10776956790540955, + "80k_to_100k": 0.03912114479997444, + "100k_to_200k": 0.055912505343134085, + "200k_plus": 0.014519794541003451 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.22536433723499413, + "mean_among_borrowers": 33402.49807525617, + "median_among_borrowers": 16000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.35882688319565326, + "mean_among_borrowers": 36572.432216778245, + "median_among_borrowers": 24900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.25221217813313995, + "mean_among_borrowers": 38788.72407799315, + "median_among_borrowers": 22900.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.13669823926255414, + "mean_among_borrowers": 40222.96469263708, + "median_among_borrowers": 25950.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03165003376804131, + "mean_among_borrowers": 50597.815332662714, + "median_among_borrowers": 37000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.156024839562724, + "mean_among_borrowers": 38475.652655313614, + "median_among_borrowers": 23182.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.19389802688054777, + "mean_among_borrowers": 40781.818553326506, + "median_among_borrowers": 25166.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.15371847235012892, + "mean_among_borrowers": 32040.145053533837, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1984583260312187, + "mean_among_borrowers": 42291.015044967404, + "median_among_borrowers": 25000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.09539828169554497, + "mean_among_borrowers": 43336.23834084036, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.16600786857086094, + "mean_among_borrowers": 39631.570218430665, + "median_among_borrowers": 21939.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.1671000542140161, + "mean_among_borrowers": 34201.14487060498, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.25267861183865514, + "mean_among_borrowers": 34586.84037560038, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.15286484625519245, + "mean_among_borrowers": 46875.77881330891, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 23668750.851507433, + "mean_balance": 36873.73724486232, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.09598168882566091, + "5k_to_10k": 0.14287136790184726, + "10k_to_20k": 0.19074970787368226, + "20k_to_40k": 0.25231020514182567, + "40k_to_60k": 0.12373308878262614, + "60k_to_80k": 0.09562201893085862, + "80k_to_100k": 0.034750948986171766, + "100k_to_200k": 0.04987747161203311, + "200k_plus": 0.014103501945294196 + }, + "age_bands": { + "24_and_younger": 0.057803501858546165, + "25_to_34": 0.35403082725820945, + "35_to_49": 0.36287233517496836, + "50_to_61": 0.1615925449287847, + "62_and_older": 0.06370079077949123 + }, + "graduate_share_of_own_original_principal": 0.45646219259032567, + "ibr_new_borrower_share": 0.5822440301014113 + } + }, + "metrics": { + "share_with_debt_error_pp": -1.4743174714625662, + "mean_among_borrowers_error_pct": -5.566593181489421, + "median_among_borrowers_error_pct": -4.815999999999998, + "household_balance_band_tvd": 0.04033667822097965, + "head_age_band_share_mae_pp": 1.3212277078973742, + "race_share_mae_pp": 2.067159177073093, + "income_quintile_share_mae_pp": 1.4743162952521935, + "borrower_balance_band_tvd": 0.05001011644613092, + "borrower_age_band_tvd": 0.023231363094113566, + "household_rank_correlation": 0.20794745847766363 + } + } + } + }, + "host": { + "release": "populace-us-2024-spm-20260915", + "path": "populace_us_2024.h5", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "is_the_published_default": true, + "households": 57240, + "people": 166321, + "seed": 0, + "gate_passed": true, + "gate_failures": [], + "household_income_columns_present": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "farm_operations_income", + "farm_rent_income", + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "rental_income", + "estate_income", + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents", + "taxable_private_pension_income", + "tax_exempt_private_pension_income", + "taxable_ira_distributions", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_sep_distributions", + "unemployment_compensation", + "alimony_income", + "child_support_received", + "veterans_benefits", + "workers_compensation", + "disability_benefits" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14412999386354491, + "mean_among_borrowers": 44483.09961323457, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 785628046119.6729, + "parent_plus_share_of_total": 0.21495679635292878 + }, + "household_balance_bands": { + "under_5k": 0.09597901426395242, + "5k_to_10k": 0.1620676904114682, + "10k_to_20k": 0.15178493618645325, + "20k_to_40k": 0.22796648343692355, + "40k_to_60k": 0.09915026443778263, + "60k_to_80k": 0.10849192930212849, + "80k_to_100k": 0.056043492646084624, + "100k_to_200k": 0.06743544596300373, + "200k_plus": 0.031080743352203163 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.2267174320421409, + "mean_among_borrowers": 18670.63953159211, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2669429791293293, + "mean_among_borrowers": 36447.86184035785, + "median_among_borrowers": 20200.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22339688829964577, + "mean_among_borrowers": 44315.447461546944, + "median_among_borrowers": 25670.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14218578094851128, + "mean_among_borrowers": 49552.354866035355, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03655929815086838, + "mean_among_borrowers": 65818.86253190483, + "median_among_borrowers": 38000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13333034770117472, + "mean_among_borrowers": 42577.59434933696, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902667788102, + "mean_among_borrowers": 47993.699863516216, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12291729609744638, + "mean_among_borrowers": 46285.94561484015, + "median_among_borrowers": 24764.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15337638725400463, + "mean_among_borrowers": 47893.571558628166, + "median_among_borrowers": 24000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12721563759800417, + "mean_among_borrowers": 38171.39283093021, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1358771691149805, + "mean_among_borrowers": 45461.58729969715, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13382646407799811, + "mean_among_borrowers": 40922.60303950533, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19993030155269514, + "mean_among_borrowers": 45004.488576349744, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12378308052696536, + "mean_among_borrowers": 52902.03665110157, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18461703.3923496, + "mean_balance": 42554.4723270352, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.1039144177980828, + "5k_to_10k": 0.16665917220324747, + "10k_to_20k": 0.15515901493993045, + "20k_to_40k": 0.23029463650667883, + "40k_to_60k": 0.10426754245310213, + "60k_to_80k": 0.09578243033185131, + "80k_to_100k": 0.05074417014558102, + "100k_to_200k": 0.06376833884555547, + "200k_plus": 0.029410276775970686 + }, + "age_bands": { + "24_and_younger": 0.03769253987750687, + "25_to_34": 0.2662405413523508, + "35_to_49": 0.39817213354338704, + "50_to_61": 0.21071813444930676, + "62_and_older": 0.08717665077744856 + }, + "graduate_share_of_own_original_principal": 0.5360724443816975, + "ibr_new_borrower_share": 0.605734686749079 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7419574258458654, + "mean_among_borrowers_error_pct": 8.357217142313411, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.061856909978440955, + "head_age_band_share_mae_pp": 3.5543373606609885, + "race_share_mae_pp": 2.8343426729118955, + "income_quintile_share_mae_pp": 4.2522526924143715, + "borrower_balance_band_tvd": 0.06971797938468582, + "borrower_age_band_tvd": 0.08466988479278444 + } + } +} From 47949111f47b81be0cb0046bc73c046ea59d0de2 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Tue, 20 Oct 2026 10:57:53 -0400 Subject: [PATCH 03/17] WIP: SCF-matching recipient definitions and #1073 host-fit diagnostics Codes the CPS recipient side the way the SCF defines it: married includes cohabiting partners, children count the partner's children, and the reference person follows the householder-couple rule. These move Parent PLUS from 21.5% to about 16% of imputed federal balances (SCF 13.8%) but leave the borrower-household share at about 14.2% (SCF 18.2%). Adds diagnosis and experiment scripts and their first receipts. The predictor-shift and remedy runs still need to run (on Modal; local memory is too low); the fit receipt and PR body are not updated yet. Part of #1073. Co-Authored-By: Claude Opus 5.5 (1M context) --- experiments/diagnose_1073_host_fit.py | 426 ++++ experiments/diagnose_1073_predictor_shift.py | 224 ++ .../explore_1073_recipient_definitions.py | 290 +++ .../receipts/1073-host-fit-diagnosis.json | 960 ++++++++ .../1073-predictor-shift-diagnosis.json | 690 ++++++ .../1073-recipient-definition-variants.json | 1972 +++++++++++++++++ .../build/us_runtime/scf_education_debt.py | 335 ++- .../us/test_us_scf_education_debt.py | 113 + 8 files changed, 4951 insertions(+), 59 deletions(-) create mode 100644 experiments/diagnose_1073_host_fit.py create mode 100644 experiments/diagnose_1073_predictor_shift.py create mode 100644 experiments/explore_1073_recipient_definitions.py create mode 100644 experiments/receipts/1073-host-fit-diagnosis.json create mode 100644 experiments/receipts/1073-predictor-shift-diagnosis.json create mode 100644 experiments/receipts/1073-recipient-definition-variants.json diff --git a/experiments/diagnose_1073_host_fit.py b/experiments/diagnose_1073_host_fit.py new file mode 100644 index 0000000000..805c1d28ca --- /dev/null +++ b/experiments/diagnose_1073_host_fit.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python +"""Diagnose the SCF-vs-CPS-host gap of the education-debt stage (microcosm#1073). + +The first receipt imputed 14.4% of CPS host households with federal education +debt against 18.2% in SCF 2025 (17.3% in the SCF hold-out), short mostly at +reference-person ages 25-49 and income quintiles 2-5, and a Parent PLUS share +of 21.5% against 13.8%. A QRF reproduces its donor's conditional distribution, +so a host-only gap means the host's predictors are distributed or coded +differently from the donor's. This script measures that, weighted, before any +change: + +1. who the reference person is (SCF: the economically dominant individual or + couple, then the male or older partner; CPS candidates: the current + oldest-adult rule and the householder-anchored rule); +2. the predictors by reference-person age band: married-or-partnered (SCF + ``married`` counts partners), children of the reference person or partner + (SCF ``kids``), race, and income summed over candidate CPS units (household, + family, SPM unit, couple plus own children, couple only), against the SCF + primary economic unit's ``wageinc`` and ``income - kginc``. + + .venv/Scripts/python.exe experiments/diagnose_1073_host_fit.py \ + --h5 populace_us_2024.h5 +""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Mapping, Sequence +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS, + load_scf_2025_education_debt_donor, +) + +AGE_BANDS = (("18_to_24", 18, 24), ("25_to_34", 25, 34), ("35_to_49", 35, 49), + ("50_to_61", 50, 61), ("62_plus", 62, 200)) # fmt: skip +_HOST_COLUMNS = ( + "person_household_id", + "person_family_id", + "person_spm_unit_id", + "age", + "is_female", + "is_household_head", + "A_LINENO", + "A_SPOUSE", + "PECOHAB", + "A_MARITL", + "PEPAR1", + "PEPAR2", + "PRDTRACE", + "PRDTHSP", + "employment_income_before_lsr", +) + + +def wq(values: np.ndarray, weights: np.ndarray, q: float) -> float: + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, q * cumulative[-1], side="left")) + return float(values[order][min(index, len(values) - 1)]) + + +def summarize(values: np.ndarray, weights: np.ndarray) -> dict[str, float]: + return { + "p25": wq(values, weights, 0.25), + "p50": wq(values, weights, 0.50), + "p75": wq(values, weights, 0.75), + "p90": wq(values, weights, 0.90), + "mean": float(np.average(values, weights=weights)), + } + + +def share(flag: np.ndarray, weights: np.ndarray) -> float: + return float(weights[flag].sum() / weights.sum()) + + +def load_host(path: Path) -> tuple[pd.DataFrame, pd.Series]: + with pd.HDFStore(path, "r") as store: + available = set(store.get_storer("/person").data_columns) + columns = [c for c in _HOST_COLUMNS if c in available] + [ + c for c in _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS if c in available + ] + person = store.select("/person", columns=columns).reset_index(drop=True) + household = store.select( + "/household", columns=["household_id", "household_weight"] + ) + weights = pd.Series( + household["household_weight"].to_numpy(dtype=float), + index=household["household_id"].to_numpy(), + ) + return person, weights + + +def _num(series: pd.Series, fill: float = 0.0) -> np.ndarray: + return pd.to_numeric(series, errors="coerce").fillna(fill).to_numpy(dtype=float) + + +def host_units(person: pd.DataFrame) -> pd.DataFrame: + """One row per household: reference person under two rules, and units.""" + + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + age = _num(person["age"]) + female = person["is_female"].astype(bool).to_numpy() + lookup = {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + spouse_ptr = _num(person["A_SPOUSE"]).astype(int) + partner_ptr = _num(person["PECOHAB"]).astype(int) + married = np.isin(_num(person["A_MARITL"]).astype(int), (1, 2)) + + def partner_of(i: int) -> int: + for pointer in (spouse_ptr[i], partner_ptr[i]): + if pointer > 0: + j = lookup.get((hh[i], int(pointer))) + if j is not None and j != i: + return j + return -1 + + # Current rule (scf_auto_loans._scf_reference_person_mask): the only adult, + # the man of a two-adult married couple (older if same sex), else the + # oldest adult. Householder rule: the CPS householder (line 1) or, when the + # householder lives with a spouse or partner, the SCF's man-or-older of the + # two. + groups = pd.Series(np.arange(len(person))).groupby(hh, sort=False) + rows = [] + for household_id, positions in groups: + idx = positions.to_numpy() + adults = idx[age[idx] >= 18] + if len(adults) == 0: + current = idx[np.argmax(age[idx])] + elif len(adults) == 1: + current = adults[0] + elif len(adults) == 2 and married[adults].all(): + if female[adults[0]] == female[adults[1]]: + current = adults[np.argmax(age[adults])] + else: + current = adults[~female[adults]][0] + else: + current = adults[np.argmax(age[adults])] + householder = idx[np.argmin(line[idx])] + partner = partner_of(householder) + anchored = householder + if partner >= 0: + pair = np.array([householder, partner]) + if female[pair[0]] != female[pair[1]]: + anchored = pair[~female[pair]][0] + else: + anchored = pair[np.argmax(age[pair])] + rows.append((household_id, current, anchored, householder)) + return pd.DataFrame( + rows, columns=["household_id", "current", "anchored", "householder"] + ) + + +def partner_rows(person: pd.DataFrame, rows: np.ndarray) -> np.ndarray: + """Positional row of each given row's spouse (A_SPOUSE) or partner (PECOHAB).""" + + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + lookup = {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + pointers = ( + _num(person["A_SPOUSE"]).astype(int), + _num(person["PECOHAB"]).astype(int), + ) + out = np.full(len(rows), -1, dtype=np.int64) + for k, r in enumerate(rows): + for ptr in pointers: + if ptr[r] > 0: + j = lookup.get((hh[r], int(ptr[r]))) + if j is not None and j != r: + out[k] = j + break + return out + + +def unit_income( + person: pd.DataFrame, + reference: np.ndarray, + unit: str, + column_sets: Mapping[str, Sequence[str]], +) -> dict[str, np.ndarray]: + """Income sums over a candidate unit around each household's reference person.""" + + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + n = len(person) + member = np.zeros(n, dtype=bool) + owner = np.full(n, -1, dtype=np.int64) # household row index of the unit + ref_rows = reference + if unit == "household": + member[:] = True + index_of = pd.Series(np.arange(len(ref_rows)), index=hh[ref_rows]) + owner = index_of.reindex(hh).to_numpy() + else: + ref_set = {} + for k, r in enumerate(ref_rows): + ref_set[r] = k + partner = partner_rows(person, ref_rows) + if unit in ("family", "spm_unit"): + key = "person_family_id" if unit == "family" else "person_spm_unit_id" + ids = person[key].to_numpy() + unit_of_ref = pd.Series(np.arange(len(ref_rows)), index=ids[ref_rows]) + unit_of_ref = unit_of_ref[~unit_of_ref.index.duplicated()] + owner = unit_of_ref.reindex(ids).to_numpy(dtype=float) + owner = np.where(np.isnan(owner), -1, owner).astype(np.int64) + # a unit id can belong to another household's reference person only + # if ids repeat across households; guard on household + same = owner >= 0 + same[same] = hh[same] == hh[ref_rows[owner[same]]] + owner = np.where(same, owner, -1) + member = owner >= 0 + if unit == "family": + # a cohabiting partner is a separate CPS family; SCF counts + # partners in the PEU, so add the partner explicitly + for k, j in enumerate(partner): + if j >= 0 and owner[j] < 0: + owner[j] = k + member[j] = True + else: + for k, r in enumerate(ref_rows): + owner[r] = k + if partner[k] >= 0: + owner[partner[k]] = k + if unit == "couple_and_children": + parents = {(hh[r], line[r]): k for k, r in enumerate(ref_rows)} + for k, j in enumerate(partner): + if j >= 0: + parents[(hh[j], line[j])] = k + for pointer in ("PEPAR1", "PEPAR2"): + ptr = _num(person[pointer]).astype(int) + for i in range(n): + if owner[i] < 0 and ptr[i] > 0: + k = parents.get((hh[i], ptr[i])) + if k is not None: + owner[i] = k + member = owner >= 0 + out = {} + for name, columns in column_sets.items(): + values = np.zeros(n) + for column in columns: + if column in person.columns: + values += _num(person[column]) + if name == "employment_income": + values = np.maximum(values, 0.0) + sums = np.zeros(len(ref_rows)) + np.add.at(sums, owner[member], values[member]) + out[name] = sums + return out + + +def children_counts( + person: pd.DataFrame, reference: np.ndarray +) -> dict[str, np.ndarray]: + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + age = _num(person["age"]) + ref_lines = {(hh[r], line[r]): k for k, r in enumerate(reference)} + partners = partner_rows(person, reference) + partner_lines = {(hh[j], line[j]): k for k, j in enumerate(partners) if j >= 0} + own = np.zeros(len(reference)) + either = np.zeros(len(reference)) + either_under_18 = np.zeros(len(reference)) + ptr1 = _num(person["PEPAR1"]).astype(int) + ptr2 = _num(person["PEPAR2"]).astype(int) + for i in range(len(person)): + keys = [(hh[i], p) for p in (ptr1[i], ptr2[i]) if p > 0] + if not keys: + continue + k_ref = next((ref_lines[x] for x in keys if x in ref_lines), None) + k_any = ( + k_ref + if k_ref is not None + else next((partner_lines[x] for x in keys if x in partner_lines), None) + ) + if k_ref is not None: + own[k_ref] += 1 + if k_any is not None: + either[k_any] += 1 + if age[i] < 18: + either_under_18[k_any] += 1 + return {"own_of_reference": own, "of_reference_or_partner": either, + "of_reference_or_partner_under_18": either_under_18} # fmt: skip + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--h5", type=Path, required=True) + parser.add_argument( + "--scf-cache-dir", type=Path, default=Path.home() / ".cache/microcosm/scf" + ) + parser.add_argument( + "--out", + type=Path, + default=Path("experiments/receipts/1073-host-fit-diagnosis.json"), + ) + args = parser.parse_args(argv) + + donor = load_scf_2025_education_debt_donor( + args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" + ) + scf_w = donor["scf_weight"].to_numpy(float) + scf_age = donor["age"].to_numpy(float) + + person, hh_weights = load_host(args.h5) + units = host_units(person) + weights = hh_weights.reindex(units["household_id"]).to_numpy(float) + age = _num(person["age"]) + report: dict[str, object] = {} + + # 1. Who is the reference person, and how old. + report["reference_person_age_bands"] = { + "scf": { + b: share((scf_age >= lo) & (scf_age <= hi), scf_w) + for b, lo, hi in AGE_BANDS + }, + **{ + f"cps_{rule}": { + b: share((age[units[rule]] >= lo) & (age[units[rule]] <= hi), weights) + for b, lo, hi in AGE_BANDS + } + for rule in ("current", "anchored") + }, + } + report["rules_disagree_share"] = share( + units["current"].to_numpy() != units["anchored"].to_numpy(), weights + ) + + column_sets = { + "employment_income": ("employment_income_before_lsr",), + "household_income": _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS, + } + for rule in ("current", "anchored"): + reference = units[rule].to_numpy() + ref_age = age[reference] + has_partner = partner_rows(person, reference) >= 0 + married_code = np.isin(_num(person["A_MARITL"]).astype(int)[reference], (1, 2)) + kids = children_counts(person, reference) + incomes = { + unit: unit_income(person, reference, unit, column_sets) + for unit in ( + "household", + "spm_unit", + "family", + "couple_and_children", + "couple", + ) + } + by_band = {} + for band, lo, hi in AGE_BANDS: + host = (ref_age >= lo) & (ref_age <= hi) + scf = (scf_age >= lo) & (scf_age <= hi) + entry = { + "scf_married_or_partnered": share( + donor["is_married"].to_numpy()[scf] == 1, scf_w[scf] + ), + "cps_A_MARITL_1_2": share(married_code[host], weights[host]), + "cps_spouse_or_partner_present": share( + has_partner[host], weights[host] + ), + "scf_kids_positive": share( + donor["own_children_in_household"].to_numpy()[scf] > 0, + scf_w[scf], + ), + **{ + f"cps_kids_{k}_positive": share(v[host] > 0, weights[host]) + for k, v in kids.items() + }, + "scf_employment_income": summarize( + donor["employment_income"].to_numpy()[scf], scf_w[scf] + ), + "scf_household_income": summarize( + donor["household_income"].to_numpy()[scf], scf_w[scf] + ), + } + for unit, sums in incomes.items(): + for name, values in sums.items(): + entry[f"cps_{name}_{unit}"] = summarize(values[host], weights[host]) + by_band[band] = entry + report[f"predictors_by_reference_age_{rule}_rule"] = by_band + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + + # Console digest + print("ref age bands", json.dumps(report["reference_person_age_bands"], indent=0)) + print("rules disagree", report["rules_disagree_share"]) + for rule in ("current", "anchored"): + for band in ("25_to_34", "35_to_49", "50_to_61"): + e = report[f"predictors_by_reference_age_{rule}_rule"][band] + print(f"\n[{rule}] {band}") + for key in ( + "scf_married_or_partnered", + "cps_A_MARITL_1_2", + "cps_spouse_or_partner_present", + "scf_kids_positive", + "cps_kids_own_of_reference_positive", + "cps_kids_of_reference_or_partner_positive", + ): + print(f" {key}: {e[key]:.3f}") + for name in ("employment_income", "household_income"): + s = e[f"scf_{name}"] + print( + f" scf {name}: p25 {s['p25']:,.0f} p50 {s['p50']:,.0f} p75 {s['p75']:,.0f} p90 {s['p90']:,.0f}" + ) + for unit in ( + "household", + "spm_unit", + "family", + "couple_and_children", + "couple", + ): + s = e[f"cps_{name}_{unit}"] + print( + f" cps {name} {unit}: p25 {s['p25']:,.0f} p50 {s['p50']:,.0f} p75 {s['p75']:,.0f} p90 {s['p90']:,.0f}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/diagnose_1073_predictor_shift.py b/experiments/diagnose_1073_predictor_shift.py new file mode 100644 index 0000000000..44384e2f2e --- /dev/null +++ b/experiments/diagnose_1073_predictor_shift.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python +"""Which predictor's SCF-vs-CPS shift drives the host borrower gap (microcosm#1073). + +Fits the stage's household-total QRF once on the SCF 2025 donor, draws the CPS +host under the stage's recipient definition, then redraws it with one +predictor at a time replaced by its SCF counterpart at the same weighted rank +within the same five-year reference-person age bin (a quantile mapping), and +with all of them replaced. A predictor whose mapping closes the gap is one the +two surveys distribute differently at equal rank. Also writes each predictor's +weighted distribution by age band. Output: +``experiments/receipts/1073-predictor-shift-diagnosis.json``. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +from collections.abc import Sequence +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + _recipients, + _stage_seed, + load_scf_2025_education_debt_donor, +) + +_HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "receipt_1073", _HERE / "build_1073_scf_education_debt_receipt.py" +) +receipt = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(receipt) + +BANDS = (("18_24", 18, 24), ("25_34", 25, 34), ("35_49", 35, 49), + ("50_61", 50, 61), ("62_plus", 62, 200)) # fmt: skip +CONTINUOUS = ( + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income", + "own_children_in_household", +) + + +def wq(values: np.ndarray, weights: np.ndarray, q: float) -> float: + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, q * cumulative[-1], side="left")) + return float(values[order][min(index, len(values) - 1)]) + + +def describe(values: np.ndarray, weights: np.ndarray, *, binary: bool) -> dict: + if binary: + return {"share": float(weights[values > 0].sum() / weights.sum())} + return { + "share_positive": float(weights[values > 0].sum() / weights.sum()), + **{ + f"p{int(q * 100)}": wq(values, weights, q) + for q in (0.25, 0.5, 0.75, 0.9, 0.99) + }, + } + + +def weighted_rank(values: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Mid-rank of each value in its weighted distribution, ties sharing a rank.""" + + frame = pd.DataFrame({"v": values, "w": weights}) + by_value = frame.groupby("v")["w"].sum().sort_index() + upper = by_value.cumsum() / by_value.sum() + mid = upper - by_value / by_value.sum() / 2.0 + return mid.reindex(values).to_numpy() + + +def quantile_map( + host: np.ndarray, + host_w: np.ndarray, + host_age: np.ndarray, + scf: np.ndarray, + scf_w: np.ndarray, + scf_age: np.ndarray, +) -> np.ndarray: + """Replace each host value by the SCF value at its rank in its age bin.""" + + out = host.copy() + bins = np.arange(15, 100, 5) + host_bin = np.digitize(host_age, bins) + scf_bin = np.digitize(scf_age, bins) + for b in np.unique(host_bin): + h = host_bin == b + s = scf_bin == b + if s.sum() < 20: + s = np.abs(scf_bin - b) <= 1 + ranks = weighted_rank(host[h], host_w[h]) + order = np.argsort(scf[s], kind="mergesort") + cumulative = np.cumsum(scf_w[s][order]) / scf_w[s].sum() + index = np.searchsorted(cumulative, ranks, side="left") + out[h] = scf[s][order][np.minimum(index, s.sum() - 1)] + return out + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--h5", type=Path, required=True) + parser.add_argument( + "--scf-cache-dir", type=Path, default=Path.home() / ".cache/microcosm/scf" + ) + parser.add_argument( + "--out", + type=Path, + default=Path("experiments/receipts/1073-predictor-shift-diagnosis.json"), + ) + args = parser.parse_args(argv) + + from microcosm.fit import QRF + + donor = load_scf_2025_education_debt_donor( + args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" + ) + frame = receipt.load_host_frame(args.h5) + host = _recipients(frame, SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION).features + host_w = np.asarray(frame.weights_for("household").values, dtype=float) + host_age = host["age"].to_numpy(float) + scf_w = donor["scf_weight"].to_numpy(float) + scf_age = donor["age"].to_numpy(float) + + report: dict[str, object] = {"distributions": {}, "counterfactual_shares": {}} + for band, lo, hi in BANDS: + h = (host_age >= lo) & (host_age <= hi) + s = (scf_age >= lo) & (scf_age <= hi) + entry = {} + for column in SCF_EDUCATION_DEBT_PREDICTORS: + if column == "age": + continue + binary = column in ("is_female", "is_married") + if column == "cps_race": + entry[column] = { + "scf": { + str(int(c)): float(scf_w[s][donor[column].to_numpy()[s] == c].sum() / scf_w[s].sum()) + for c in (1, 2, 3, 4, 7) + }, + "cps": { + str(int(c)): float(host_w[h][host[column].to_numpy()[h] == c].sum() / host_w[h].sum()) + for c in (1, 2, 3, 4, 7) + }, + } # fmt: skip + continue + entry[column] = { + "scf": describe(donor[column].to_numpy(float)[s], scf_w[s], binary=binary), + "cps": describe(host[column].to_numpy(float)[h], host_w[h], binary=binary), + } # fmt: skip + report["distributions"][band] = entry + + predictors = list(SCF_EDUCATION_DEBT_PREDICTORS) + fit = donor.loc[:, [*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET, "scf_weight"]] + model = QRF(n_estimators=100, seed=_stage_seed(0, 1)).fit( + fit, + predictors=predictors, + targets=[SCF_EDUCATION_DEBT_TOTAL_TARGET], + weights="scf_weight", + ) + + def shares(features: pd.DataFrame) -> dict[str, float]: + drawn = model.predict(features.loc[:, predictors])[ + SCF_EDUCATION_DEBT_TOTAL_TARGET + ].to_numpy() + positive = drawn > 0 + out = {"all": float(host_w[positive].sum() / host_w.sum())} + for band, lo, hi in BANDS: + b = (host_age >= lo) & (host_age <= hi) + out[band] = float(host_w[b & positive].sum() / host_w[b].sum()) + return out + + scf_positive = donor[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy() > 0 + report["scf_observed_shares"] = { + "all": float(scf_w[scf_positive].sum() / scf_w.sum()), + **{ + band: float( + scf_w[scf_positive & (scf_age >= lo) & (scf_age <= hi)].sum() + / scf_w[(scf_age >= lo) & (scf_age <= hi)].sum() + ) + for band, lo, hi in BANDS + }, + } + report["counterfactual_shares"]["host_as_is"] = shares(host) + mapped_all = host.copy() + for column in CONTINUOUS: + mapped = host.copy() + mapped[column] = quantile_map( + host[column].to_numpy(float), + host_w, + host_age, + donor[column].to_numpy(float), + scf_w, + scf_age, + ) + mapped_all[column] = mapped[column] + report["counterfactual_shares"][f"map_{column}"] = shares(mapped) + print( + column, + json.dumps(report["counterfactual_shares"][f"map_{column}"]), + flush=True, + ) + report["counterfactual_shares"]["map_all_continuous"] = shares(mapped_all) + print("host", json.dumps(report["counterfactual_shares"]["host_as_is"])) + print("all", json.dumps(report["counterfactual_shares"]["map_all_continuous"])) + print("scf", json.dumps(report["scf_observed_shares"])) + for band in ("25_34", "35_49"): + print(band, json.dumps(report["distributions"][band], default=float)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps(report, indent=2, default=float) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/explore_1073_recipient_definitions.py b/experiments/explore_1073_recipient_definitions.py new file mode 100644 index 0000000000..8840a228cf --- /dev/null +++ b/experiments/explore_1073_recipient_definitions.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +"""Host fit under each CPS construction of the SCF predictors (microcosm#1073). + +Each variant changes one field of ``RecipientDefinition`` from the one before, +for a stated reason (``VARIANTS``), and imputes the published default with the +same seed and donor through the production code path. Writes +``experiments/receipts/1073-recipient-definition-variants.json``. + + .venv/Scripts/python.exe experiments/explore_1073_recipient_definitions.py \ + --h5 populace_us_2024.h5 +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +from collections.abc import Sequence +from dataclasses import asdict, replace +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + RecipientDefinition, + _recipients, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, +) +from microcosm.frame import Frame + +_HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "receipt_1073", _HERE / "build_1073_scf_education_debt_receipt.py" +) +receipt = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(receipt) + +FIRST = SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION +_V1 = replace(FIRST, partnered="spouse_or_partner") +_V2 = replace(_V1, children="of_reference_or_partner") +_V3 = replace(_V2, reference_person="householder_couple") +VARIANTS: tuple[tuple[str, RecipientDefinition, str], ...] = ( + ("v0_first_receipt", FIRST, "The first receipt's construction."), + ( + "v1_partner_counts_as_married", + _V1, + "SCF MARRIED is married or living with a partner (X8023 in 1-2); " + "A_MARITL 1-2 drops cohabiting partners.", + ), + ( + "v2_children_of_reference_or_partner", + _V2, + "SCF KIDS counts children of the reference person or spouse/partner, " + "any age, anywhere in the household listing.", + ), + ( + "v3_householder_couple_reference_person", + _V3, + "The SCF reference person comes from the economically dominant " + "individual or couple (the man, or the older partner); the CPS " + "householder anchors that unit, where the oldest adult can be a " + "co-resident parent or roommate.", + ), + ( + "v4a_spm_unit_income", + replace(_V3, income_unit="spm_unit"), + "SCF income covers the primary economic unit; the SPM unit is the CPS " + "resource-sharing unit.", + ), + ( + "v4b_couple_and_children_income", + replace(_V3, income_unit="reference_couple_and_children"), + "The PEU narrowed to the couple and their children.", + ), +) +_ALL = tuple(SCF_EDUCATION_DEBT_PREDICTORS) +_NO_INTDIV = tuple(p for p in _ALL if p != "interest_dividend_income") +_NO_INTDIV_SSRET = tuple(p for p in _NO_INTDIV if p != "social_security_pension_income") +#: Predictor-set remedies on top of the couple income unit. Each row also gets +#: an SCF hold-out so the dropped predictor's own contribution is measured. +PREDICTOR_VARIANTS: tuple[ + tuple[str, RecipientDefinition, tuple[str, ...], str], ... +] = ( + ( + "r1_couple_income_drop_interest_dividends", + replace(_V3, income_unit="reference_couple_and_children"), + _NO_INTDIV, + "CPS interest and dividends are PUF-imputed at tax-unit level; at " + "reference ages 25-34, 42% of host households have some against 16% " + "of SCF families, so equal values do not mean equal positions.", + ), + ( + "r2_couple_income_drop_interest_dividends_and_ss_pension", + replace(_V3, income_unit="reference_couple_and_children"), + _NO_INTDIV_SSRET, + "As r1, and Social Security or pension income: 18% of host households " + "at 25-34 have some against 6% of SCF families.", + ), + ( + "r3_household_income_drop_interest_dividends_and_ss_pension", + _V3, + _NO_INTDIV_SSRET, + "As r2 with the household income unit, to separate the two changes.", + ), +) + + +def parent_plus_by_age( + head_age: np.ndarray, parent_plus: np.ndarray, total: np.ndarray, weight: np.ndarray +) -> dict[str, float]: + out = {} + for label, low, high in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: + inside = (head_age >= low) & (head_age <= high) + denominator = float(np.dot(total[inside], weight[inside])) + out[label] = ( + float(np.dot(parent_plus[inside], weight[inside])) / denominator + if denominator > 0 + else float("nan") + ) + return out + + +def host_parent_plus_by_age(frame: Frame, definition: RecipientDefinition) -> dict: + person = frame.table("person") + recipients = _recipients(frame, definition) + household_ids = person["person_household_id"].to_numpy() + order = frame.table("household")["household_id"].to_numpy() + own = person["federal_student_loan_balance"].to_numpy(dtype=float) + plus = person["federal_parent_plus_loan_balance"].to_numpy(dtype=float) + total = ( + pd.Series(own + plus, index=household_ids) + .groupby(level=0) + .sum() + .reindex(order) + .to_numpy() + ) + plus_hh = ( + pd.Series(plus, index=household_ids).groupby(level=0).sum().reindex(order) + ).to_numpy() + return parent_plus_by_age( + recipients.features["age"].to_numpy(dtype=float), + plus_hh, + total, + np.asarray(frame.weights_for("household").values, dtype=float), + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--h5", type=Path, required=True) + parser.add_argument( + "--scf-cache-dir", type=Path, default=Path.home() / ".cache/microcosm/scf" + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--n-estimators", type=int, default=100) + parser.add_argument( + "--out", + type=Path, + default=Path("experiments/receipts/1073-recipient-definition-variants.json"), + ) + args = parser.parse_args(argv) + + donor = load_scf_2025_education_debt_donor( + args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" + ) + observed = scf_education_debt_donor_fit_table(donor) + scf_parent_plus = parent_plus_by_age( + donor["age"].to_numpy(float), + donor["federal_parent_plus_loan_balance"].to_numpy(float), + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(float), + donor["scf_weight"].to_numpy(float), + ) + base = receipt.load_host_frame(args.h5) + results = { + "scf_2025": { + "share": observed["households"]["share_with_federal_education_debt"], + "parent_plus_share": observed["households"]["parent_plus_share_of_total"], + "parent_plus_by_reference_age": scf_parent_plus, + "by_head_age_band": { + k: v["share_with_federal_education_debt"] + for k, v in observed["by_head_age_band"].items() + }, + "by_income_quintile": { + k: v["share_with_federal_education_debt"] + for k, v in observed["by_income_quintile"].items() + }, + }, + "variants": {}, + } + holdout: dict[tuple[str, ...], dict] = {} + runs = [(n, d, _ALL, r) for n, d, r in VARIANTS] + list(PREDICTOR_VARIANTS) + for name, definition, predictors, reason in runs: + if predictors not in holdout: + drawn = receipt.holdout_draws( + donor, + predictors, + folds=5, + seed=args.seed, + n_estimators=args.n_estimators, + ) + holdout_fit = scf_education_debt_donor_fit_table(donor, imputed=drawn) + holdout[predictors] = { + "predictors": list(predictors), + "share": holdout_fit["households"]["share_with_federal_education_debt"], + "parent_plus_share": holdout_fit["households"][ + "parent_plus_share_of_total" + ], + "metrics_vs_scf": receipt.fit_metrics(observed, holdout_fit), + } + print("holdout", json.dumps(holdout[predictors], default=float), flush=True) + imputed = impute_us_scf_education_debt( + base, + donor, + seed=args.seed, + n_estimators=args.n_estimators, + definition=definition, + predictors=predictors, + ) + tables = {entity: base.table(entity).copy() for entity in base.entities} + for column in imputed.columns: + tables["person"][column] = imputed[column].to_numpy() + frame = Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + ) + fit = us_scf_education_debt_fit_table(frame, definition) + metrics = receipt.fit_metrics(observed, fit) + entry = { + "definition": asdict(definition), + "predictors": list(predictors), + "scf_holdout": holdout[predictors], + "reason": reason, + "share": fit["households"]["share_with_federal_education_debt"], + "parent_plus_share": fit["households"]["parent_plus_share_of_total"], + "parent_plus_by_reference_age": host_parent_plus_by_age(frame, definition), + "by_head_age_band": { + k: v["share_with_federal_education_debt"] + for k, v in fit["by_head_age_band"].items() + }, + "by_income_quintile": { + k: v["share_with_federal_education_debt"] + for k, v in fit["by_income_quintile"].items() + }, + "metrics_vs_scf": metrics, + "fit": fit, + } + results["variants"][name] = entry + print( + name, + json.dumps( + { + "share": round(entry["share"], 4), + "pp": round(entry["parent_plus_share"], 4), + "age": { + k: round(v, 3) for k, v in entry["by_head_age_band"].items() + }, + "inc": { + k: round(v, 3) for k, v in entry["by_income_quintile"].items() + }, + "pp_age": { + k: round(v, 3) + for k, v in entry["parent_plus_by_reference_age"].items() + }, + "age_mae": round(metrics["head_age_band_share_mae_pp"], 2), + "inc_mae": round(metrics["income_quintile_share_mae_pp"], 2), + } + ), + flush=True, + ) + print("scf", json.dumps(results["scf_2025"], default=float)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps(results, indent=2, default=float) + "\n", + encoding="utf-8", + newline="\n", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/receipts/1073-host-fit-diagnosis.json b/experiments/receipts/1073-host-fit-diagnosis.json new file mode 100644 index 0000000000..4251dbea75 --- /dev/null +++ b/experiments/receipts/1073-host-fit-diagnosis.json @@ -0,0 +1,960 @@ +{ + "reference_person_age_bands": { + "scf": { + "18_to_24": 0.04337107980722341, + "25_to_34": 0.15258919561765283, + "35_to_49": 0.2552374964878215, + "50_to_61": 0.1954852936381943, + "62_plus": 0.353316934449108 + }, + "cps_current": { + "18_to_24": 0.02144748678037467, + "25_to_34": 0.14294679276322914, + "35_to_49": 0.2526892646920257, + "50_to_61": 0.22104134738832001, + "62_plus": 0.3618751083760504 + }, + "cps_anchored": { + "18_to_24": 0.0339806407185135, + "25_to_34": 0.16053637335441817, + "35_to_49": 0.2644394013805101, + "50_to_61": 0.2100486920170891, + "62_plus": 0.32936379309579705 + } + }, + "rules_disagree_share": 0.13144464121075464, + "predictors_by_reference_age_current_rule": { + "18_to_24": { + "scf_married_or_partnered": 0.388466732733208, + "cps_A_MARITL_1_2": 0.3359212024332317, + "cps_spouse_or_partner_present": 0.3480042049290887, + "scf_kids_positive": 0.24031884047789556, + "cps_kids_own_of_reference_positive": 0.3147757964709492, + "cps_kids_of_reference_or_partner_positive": 0.3167990656849633, + "cps_kids_of_reference_or_partner_under_18_positive": 0.3167989510144334, + "scf_employment_income": { + "p25": 13351.351351351352, + "p50": 27729.72972972973, + "p75": 48270.27027027027, + "p90": 82162.16216216216, + "mean": 35679.59226261271 + }, + "scf_household_income": { + "p25": 21567.567567567567, + "p50": 38000.0, + "p75": 64682.16216216216, + "p90": 87297.2972972973, + "mean": 46821.884787205934 + }, + "cps_employment_income_household": { + "p25": 19942.932160142125, + "p50": 38000.0, + "p75": 65408.06916808518, + "p90": 86000.0, + "mean": 52431.37768321981 + }, + "cps_household_income_household": { + "p25": 25000.0, + "p50": 48400.0, + "p75": 81039.91162936026, + "p90": 121000.0, + "mean": 67888.96741859884 + }, + "cps_employment_income_spm_unit": { + "p25": 18043.605287747636, + "p50": 30033.106169737843, + "p75": 56000.0, + "p90": 77991.10969769866, + "mean": 46731.39114091128 + }, + "cps_household_income_spm_unit": { + "p25": 22000.0, + "p50": 45845.46664153128, + "p75": 75295.16341639643, + "p90": 103393.52709145591, + "mean": 61502.589525727824 + }, + "cps_employment_income_family": { + "p25": 18000.0, + "p50": 30000.0, + "p75": 50450.870047978584, + "p90": 70037.67841954675, + "mean": 43348.42049301801 + }, + "cps_household_income_family": { + "p25": 22000.0, + "p50": 40000.0, + "p75": 67600.0, + "p90": 88126.78110726044, + "mean": 55391.41494865657 + }, + "cps_employment_income_couple_and_children": { + "p25": 18000.0, + "p50": 30000.0, + "p75": 48400.0, + "p90": 70037.67841954675, + "mean": 43022.594188614865 + }, + "cps_household_income_couple_and_children": { + "p25": 21960.966962061266, + "p50": 38000.0, + "p75": 66990.71639962481, + "p90": 88126.78110726044, + "mean": 55065.069122927605 + }, + "cps_employment_income_couple": { + "p25": 18000.0, + "p50": 30000.0, + "p75": 48400.0, + "p90": 70037.67841954675, + "mean": 43022.56840071297 + }, + "cps_household_income_couple": { + "p25": 21960.966962061266, + "p50": 38000.0, + "p75": 66990.71639962481, + "p90": 88126.78110726044, + "mean": 55065.04193637662 + } + }, + "25_to_34": { + "scf_married_or_partnered": 0.5399894876417738, + "cps_A_MARITL_1_2": 0.4194186944982328, + "cps_spouse_or_partner_present": 0.4739143575211383, + "scf_kids_positive": 0.3846077052668725, + "cps_kids_own_of_reference_positive": 0.4335671738654255, + "cps_kids_of_reference_or_partner_positive": 0.43783228293988347, + "cps_kids_of_reference_or_partner_under_18_positive": 0.43717205052775054, + "scf_employment_income": { + "p25": 36972.97297297297, + "p50": 71891.8918918919, + "p75": 114000.0, + "p90": 174594.5945945946, + "mean": 85191.95501086062 + }, + "scf_household_income": { + "p25": 46216.21621621621, + "p50": 82162.16216216216, + "p75": 125297.29729729731, + "p90": 184864.86486486485, + "mean": 99989.46215041565 + }, + "cps_employment_income_household": { + "p25": 30033.106169737843, + "p50": 60000.0, + "p75": 108024.2158674365, + "p90": 179961.22115937775, + "mean": 83427.50903540682 + }, + "cps_household_income_household": { + "p25": 38342.35867099277, + "p50": 70000.0, + "p75": 122333.26248998212, + "p90": 198027.7607303203, + "mean": 97759.4912193431 + }, + "cps_employment_income_spm_unit": { + "p25": 29000.0, + "p50": 55000.0, + "p75": 98000.0, + "p90": 160000.0, + "mean": 76987.71046452793 + }, + "cps_household_income_spm_unit": { + "p25": 35968.50264597061, + "p50": 64407.40279386099, + "p75": 113400.0, + "p90": 183358.8295516743, + "mean": 89678.6019096458 + }, + "cps_employment_income_family": { + "p25": 25937.682601137225, + "p50": 50000.0, + "p75": 89980.61057968887, + "p90": 150046.82291916455, + "mean": 71645.64914387572 + }, + "cps_household_income_family": { + "p25": 31447.37312970231, + "p50": 59947.50440995103, + "p75": 103687.45621135348, + "p90": 168766.56367351895, + "mean": 83839.69847403152 + }, + "cps_employment_income_couple_and_children": { + "p25": 25047.37312970231, + "p50": 49976.038329879964, + "p75": 87962.57577776973, + "p90": 150046.82291916455, + "mean": 70642.49060993642 + }, + "cps_household_income_couple_and_children": { + "p25": 30034.085403312194, + "p50": 58779.602609256326, + "p75": 101685.40205932752, + "p90": 164647.15169238666, + "mean": 82573.8872383556 + }, + "cps_employment_income_couple": { + "p25": 25047.37312970231, + "p50": 49976.038329879964, + "p75": 87962.57577776973, + "p90": 150000.0, + "mean": 70591.86990128312 + }, + "cps_household_income_couple": { + "p25": 30033.106169737843, + "p50": 58779.602609256326, + "p75": 101685.40205932752, + "p90": 164647.15169238666, + "mean": 82504.91553190212 + } + }, + "35_to_49": { + "scf_married_or_partnered": 0.6290734845229035, + "cps_A_MARITL_1_2": 0.6373316031380384, + "cps_spouse_or_partner_present": 0.6612862067465197, + "scf_kids_positive": 0.6816213933375661, + "cps_kids_own_of_reference_positive": 0.6975751973361995, + "cps_kids_of_reference_or_partner_positive": 0.6991037252799116, + "cps_kids_of_reference_or_partner_under_18_positive": 0.6415944881431738, + "scf_employment_income": { + "p25": 34918.91891891892, + "p50": 82162.16216216216, + "p75": 143783.7837837838, + "p90": 225945.94594594595, + "mean": 114824.6600335461 + }, + "scf_household_income": { + "p25": 51351.35135135135, + "p50": 100648.64864864864, + "p75": 169459.45945945944, + "p90": 262364.3243243243, + "mean": 143147.9300195838 + }, + "cps_employment_income_household": { + "p25": 33475.63612595285, + "p50": 75023.41145958228, + "p75": 134100.0, + "p90": 219965.79340918665, + "mean": 107445.50962305516 + }, + "cps_household_income_household": { + "p25": 44929.96486229587, + "p50": 93715.00896244941, + "p75": 162664.02047770523, + "p90": 259851.65772947093, + "mean": 130842.88650477315 + }, + "cps_employment_income_spm_unit": { + "p25": 32419.13555318342, + "p50": 75000.0, + "p75": 131200.0, + "p90": 219965.79340918665, + "mean": 106002.14996021929 + }, + "cps_household_income_spm_unit": { + "p25": 43130.183185627066, + "p50": 91944.54097585058, + "p75": 160374.41278780956, + "p90": 258082.52630531235, + "mean": 129029.41398362057 + }, + "cps_employment_income_family": { + "p25": 30033.106169737843, + "p50": 71462.1735738426, + "p75": 130000.0, + "p90": 219016.12997298938, + "mean": 103694.39469413263 + }, + "cps_household_income_family": { + "p25": 40435.91003146334, + "p50": 90000.0, + "p75": 157623.70447324187, + "p90": 254280.3863446263, + "mean": 126358.13459474227 + }, + "cps_employment_income_couple_and_children": { + "p25": 30000.0, + "p50": 70037.67841954675, + "p75": 129985.18282949777, + "p90": 217116.8031005949, + "mean": 102415.8512812932 + }, + "cps_household_income_couple_and_children": { + "p25": 40000.0, + "p50": 89000.0, + "p75": 156473.66496267804, + "p90": 252192.08087394285, + "mean": 124568.85845079289 + }, + "cps_employment_income_couple": { + "p25": 27000.0, + "p50": 65051.94537951121, + "p75": 127000.0, + "p90": 209994.32732911556, + "mean": 98762.48810456705 + }, + "cps_household_income_couple": { + "p25": 37036.87401169252, + "p50": 83927.75612929699, + "p75": 149636.9018044752, + "p90": 243508.67594867502, + "mean": 119921.62423133879 + } + }, + "50_to_61": { + "scf_married_or_partnered": 0.5823377506882798, + "cps_A_MARITL_1_2": 0.6827162972424433, + "cps_spouse_or_partner_present": 0.6952105600998268, + "scf_kids_positive": 0.4757664892740851, + "cps_kids_own_of_reference_positive": 0.5800850452908032, + "cps_kids_of_reference_or_partner_positive": 0.5808386543289418, + "cps_kids_of_reference_or_partner_under_18_positive": 0.21670674321239264, + "scf_employment_income": { + "p25": 23621.62162162162, + "p50": 71891.8918918919, + "p75": 159189.1891891892, + "p90": 259837.83783783787, + "mean": 130259.85946328501 + }, + "scf_household_income": { + "p25": 48270.27027027027, + "p50": 100648.64864864865, + "p75": 186918.9189189189, + "p90": 328648.64864864864, + "mean": 180220.35393913506 + }, + "cps_employment_income_household": { + "p25": 40000.0, + "p50": 82900.0, + "p75": 155000.0, + "p90": 249998.89957892447, + "mean": 123769.04398414477 + }, + "cps_household_income_household": { + "p25": 63163.951453154354, + "p50": 120013.7167494267, + "p75": 199458.61673261435, + "p90": 314306.98515674786, + "mean": 164360.507613877 + }, + "cps_employment_income_spm_unit": { + "p25": 38700.0, + "p50": 81000.0, + "p75": 153014.52115728092, + "p90": 246200.0, + "mean": 122021.94149808068 + }, + "cps_household_income_spm_unit": { + "p25": 61200.0, + "p50": 117634.69211716403, + "p75": 197193.69123772596, + "p90": 312421.4000230408, + "mean": 162408.58786822122 + }, + "cps_employment_income_family": { + "p25": 37749.12158884045, + "p50": 80000.0, + "p75": 151471.31807346042, + "p90": 245036.90812479387, + "mean": 120510.2492529643 + }, + "cps_household_income_family": { + "p25": 59947.50440995103, + "p50": 114954.63160279387, + "p75": 194167.9995137999, + "p90": 312000.0, + "mean": 160553.95287265073 + }, + "cps_employment_income_couple_and_children": { + "p25": 36324.62643454458, + "p50": 77991.10969769866, + "p75": 150000.0, + "p90": 240027.4334988534, + "mean": 118613.8043169604 + }, + "cps_household_income_couple_and_children": { + "p25": 58000.0, + "p50": 113020.03074088796, + "p75": 192230.31038857266, + "p90": 310440.62512728525, + "mean": 158301.42155607024 + }, + "cps_employment_income_couple": { + "p25": 28500.0, + "p50": 65051.94537951121, + "p75": 129985.18282949776, + "p90": 216641.9713824963, + "mean": 105803.30851218582 + }, + "cps_household_income_couple": { + "p25": 47360.0, + "p50": 95849.08808532642, + "p75": 170175.8631001535, + "p90": 285000.0, + "mean": 142740.79730197633 + } + }, + "62_plus": { + "scf_married_or_partnered": 0.5111385359889865, + "cps_A_MARITL_1_2": 0.7019606119626662, + "cps_spouse_or_partner_present": 0.7078680807763289, + "scf_kids_positive": 0.15155608421924158, + "cps_kids_own_of_reference_positive": 0.29937383262058725, + "cps_kids_of_reference_or_partner_positive": 0.30075570097634075, + "cps_kids_of_reference_or_partner_under_18_positive": 0.027736868812659743, + "scf_employment_income": { + "p25": 0.0, + "p50": 0.0, + "p75": 33891.89189189189, + "p90": 103729.72972972973, + "mean": 40191.80953371061 + }, + "scf_household_income": { + "p25": 36972.97297297297, + "p50": 69837.83783783784, + "p75": 128378.37837837837, + "p90": 237243.24324324325, + "mean": 128923.04558037713 + }, + "cps_employment_income_household": { + "p25": 0.0, + "p50": 7003.7678419546755, + "p75": 60000.0, + "p90": 155000.0, + "mean": 55427.92717467862 + }, + "cps_household_income_household": { + "p25": 49926.53540083874, + "p50": 85653.87059191501, + "p75": 154569.5722100861, + "p90": 253441.3565921235, + "mean": 127766.85179782026 + }, + "cps_employment_income_spm_unit": { + "p25": 0.0, + "p50": 6000.0, + "p75": 59947.50440995103, + "p90": 151150.80666374386, + "mean": 54474.78102380924 + }, + "cps_household_income_spm_unit": { + "p25": 49150.3533963364, + "p50": 84712.14478274979, + "p75": 152507.42901065134, + "p90": 252155.32989647862, + "mean": 126486.6251861578 + }, + "cps_employment_income_family": { + "p25": 0.0, + "p50": 5000.0, + "p75": 57600.0, + "p90": 150046.82291916455, + "mean": 53421.41217379493 + }, + "cps_household_income_family": { + "p25": 48141.095029587144, + "p50": 83336.07663954931, + "p75": 150534.97679427481, + "p90": 250353.52248550823, + "mean": 125251.01837523092 + }, + "cps_employment_income_couple_and_children": { + "p25": 0.0, + "p50": 2896.473480401594, + "p75": 50000.0, + "p90": 134970.91586953326, + "mean": 47871.698899062045 + }, + "cps_household_income_couple_and_children": { + "p25": 46615.94538092046, + "p50": 80271.70886805636, + "p75": 138278.80882439725, + "p90": 231474.6077227419, + "mean": 118040.88028071057 + }, + "cps_employment_income_couple": { + "p25": 0.0, + "p50": 0.0, + "p75": 30000.0, + "p90": 99952.07665975994, + "mean": 36870.9334410469 + }, + "cps_household_income_couple": { + "p25": 39077.49835560404, + "p50": 67652.4092674109, + "p75": 118968.15700749046, + "p90": 200189.31458618637, + "mean": 103452.06921503063 + } + } + }, + "predictors_by_reference_age_anchored_rule": { + "18_to_24": { + "scf_married_or_partnered": 0.388466732733208, + "cps_A_MARITL_1_2": 0.22474301409043093, + "cps_spouse_or_partner_present": 0.24320062204068002, + "scf_kids_positive": 0.24031884047789556, + "cps_kids_own_of_reference_positive": 0.25224238041401326, + "cps_kids_of_reference_or_partner_positive": 0.2522423862320045, + "cps_kids_of_reference_or_partner_under_18_positive": 0.25224228725049114, + "scf_employment_income": { + "p25": 13351.351351351352, + "p50": 27729.72972972973, + "p75": 48270.27027027027, + "p90": 82162.16216216216, + "mean": 35679.59226261271 + }, + "scf_household_income": { + "p25": 21567.567567567567, + "p50": 38000.0, + "p75": 64682.16216216216, + "p90": 87297.2972972973, + "mean": 46821.884787205934 + }, + "cps_employment_income_household": { + "p25": 23040.0, + "p50": 46200.0, + "p75": 77991.10969769866, + "p90": 112891.24097794737, + "mean": 62177.78547112806 + }, + "cps_household_income_household": { + "p25": 31220.185464984395, + "p50": 58862.3520972485, + "p75": 88793.53128444232, + "p90": 144160.0, + "mean": 78217.66710942617 + }, + "cps_employment_income_spm_unit": { + "p25": 19942.932160142125, + "p50": 41000.0, + "p75": 70037.67841954675, + "p90": 94966.34361972439, + "mean": 55282.881554043466 + }, + "cps_household_income_spm_unit": { + "p25": 25182.020977098553, + "p50": 49026.37489368272, + "p75": 83680.04338189503, + "p90": 135237.34235554864, + "mean": 70153.52360070287 + }, + "cps_employment_income_family": { + "p25": 19942.932160142125, + "p50": 35000.0, + "p75": 59295.0, + "p90": 90000.0, + "mean": 51824.32925091325 + }, + "cps_household_income_family": { + "p25": 23160.0, + "p50": 46000.0, + "p75": 82868.14092869547, + "p90": 124999.44978946223, + "mean": 66329.47651142394 + }, + "cps_employment_income_couple_and_children": { + "p25": 14957.199120106592, + "p50": 23979.00176398041, + "p75": 44990.30528984444, + "p90": 65408.06916808518, + "mean": 36492.192696942155 + }, + "cps_household_income_couple_and_children": { + "p25": 17000.0, + "p50": 30000.0, + "p75": 54749.00186433036, + "p90": 85662.03338928835, + "mean": 46503.9521863318 + }, + "cps_employment_income_couple": { + "p25": 14957.199120106592, + "p50": 23979.00176398041, + "p75": 44990.30528984444, + "p90": 65408.06916808518, + "mean": 36492.02322061497 + }, + "cps_household_income_couple": { + "p25": 17000.0, + "p50": 30000.0, + "p75": 54749.00186433036, + "p90": 85662.03338928835, + "mean": 46503.77960773607 + } + }, + "25_to_34": { + "scf_married_or_partnered": 0.5399894876417738, + "cps_A_MARITL_1_2": 0.39715604985032005, + "cps_spouse_or_partner_present": 0.452259012174844, + "scf_kids_positive": 0.3846077052668725, + "cps_kids_own_of_reference_positive": 0.42409872327777987, + "cps_kids_of_reference_or_partner_positive": 0.4305778569704265, + "cps_kids_of_reference_or_partner_under_18_positive": 0.42916253321052183, + "scf_employment_income": { + "p25": 36972.97297297297, + "p50": 71891.8918918919, + "p75": 114000.0, + "p90": 174594.5945945946, + "mean": 85191.95501086062 + }, + "scf_household_income": { + "p25": 46216.21621621621, + "p50": 82162.16216216216, + "p75": 125297.29729729731, + "p90": 184864.86486486485, + "mean": 99989.46215041565 + }, + "cps_employment_income_household": { + "p25": 33000.0, + "p50": 62202.95507091948, + "p75": 113100.0, + "p90": 187083.69693085706, + "mean": 88380.5525425812 + }, + "cps_household_income_household": { + "p25": 41073.922849105154, + "p50": 75949.0, + "p75": 133000.0, + "p90": 219819.68333705288, + "mean": 105277.24994559937 + }, + "cps_employment_income_spm_unit": { + "p25": 30033.106169737843, + "p50": 59947.50440995103, + "p75": 104500.0, + "p90": 174975.4881193422, + "mean": 82239.9343919021 + }, + "cps_household_income_spm_unit": { + "p25": 38160.0, + "p50": 70000.0, + "p75": 121174.55918696911, + "p90": 199353.78844982432, + "mean": 97549.04089538947 + }, + "cps_employment_income_family": { + "p25": 28566.0, + "p50": 54961.77136991549, + "p75": 95000.0, + "p90": 160018.2889992356, + "mean": 76157.89229740339 + }, + "cps_household_income_family": { + "p25": 35604.32233085497, + "p50": 64355.04341526062, + "p75": 112703.91751543077, + "p90": 184869.72855043667, + "mean": 90541.35486780162 + }, + "cps_employment_income_couple_and_children": { + "p25": 25000.0, + "p50": 47958.00352796082, + "p75": 85000.0, + "p90": 148000.0, + "mean": 68543.22340338948 + }, + "cps_household_income_couple_and_children": { + "p25": 30033.106169737843, + "p50": 57454.63788993326, + "p75": 101001.0, + "p90": 161218.58424981666, + "mean": 80561.55919990591 + }, + "cps_employment_income_couple": { + "p25": 25000.0, + "p50": 47958.00352796082, + "p75": 85000.0, + "p90": 147435.24846962214, + "mean": 68492.34825035393 + }, + "cps_household_income_couple": { + "p25": 30033.106169737843, + "p50": 57335.929960408605, + "p75": 100859.32026419185, + "p90": 161218.58424981666, + "mean": 80180.74299481639 + } + }, + "35_to_49": { + "scf_married_or_partnered": 0.6290734845229035, + "cps_A_MARITL_1_2": 0.6440453902603906, + "cps_spouse_or_partner_present": 0.6653514555372678, + "scf_kids_positive": 0.6816213933375661, + "cps_kids_own_of_reference_positive": 0.7030707373318383, + "cps_kids_of_reference_or_partner_positive": 0.7049748206449906, + "cps_kids_of_reference_or_partner_under_18_positive": 0.644465986933246, + "scf_employment_income": { + "p25": 34918.91891891892, + "p50": 82162.16216216216, + "p75": 143783.7837837838, + "p90": 225945.94594594595, + "mean": 114824.6600335461 + }, + "scf_household_income": { + "p25": 51351.35135135135, + "p50": 100648.64864864864, + "p75": 169459.45945945944, + "p90": 262364.3243243243, + "mean": 143147.9300195838 + }, + "cps_employment_income_household": { + "p25": 32644.68061928026, + "p50": 75973.07489577951, + "p75": 139956.64890956884, + "p90": 220559.3330568099, + "mean": 109366.46309466714 + }, + "cps_household_income_household": { + "p25": 46409.00779119151, + "p50": 96035.69421902066, + "p75": 167809.5201741723, + "p90": 275503.0, + "mean": 135418.09577280766 + }, + "cps_employment_income_spm_unit": { + "p25": 32000.0, + "p50": 75023.41145958228, + "p75": 138057.3220371743, + "p90": 220000.0, + "mean": 108013.05830095464 + }, + "cps_household_income_spm_unit": { + "p25": 45285.13932898871, + "p50": 95000.0, + "p75": 165710.83040519131, + "p90": 272232.17023368436, + "mean": 133815.90995804372 + }, + "cps_employment_income_family": { + "p25": 30000.0, + "p50": 72767.96079861381, + "p75": 134021.25243333605, + "p90": 219965.79340918665, + "mean": 105409.16708978848 + }, + "cps_household_income_family": { + "p25": 42858.65635704456, + "p50": 92097.22979731677, + "p75": 163342.45075975615, + "p90": 269440.6022459578, + "mean": 130881.01146723755 + }, + "cps_employment_income_couple_and_children": { + "p25": 28990.0, + "p50": 70037.67841954675, + "p75": 131000.0, + "p90": 215000.0, + "mean": 102739.63122531319 + }, + "cps_household_income_couple_and_children": { + "p25": 39648.44846123493, + "p50": 89000.0, + "p75": 157340.22456503127, + "p90": 254280.3863446263, + "mean": 125446.97178857018 + }, + "cps_employment_income_couple": { + "p25": 26946.700002096797, + "p50": 65594.0, + "p75": 128000.0, + "p90": 207976.29252719643, + "mean": 99339.73185688264 + }, + "cps_household_income_couple": { + "p25": 37726.0, + "p50": 85190.84216959236, + "p75": 151739.87344439077, + "p90": 243819.0965940156, + "mean": 121211.93151601273 + } + }, + "50_to_61": { + "scf_married_or_partnered": 0.5823377506882798, + "cps_A_MARITL_1_2": 0.6976656598435952, + "cps_spouse_or_partner_present": 0.7105272865424017, + "scf_kids_positive": 0.4757664892740851, + "cps_kids_own_of_reference_positive": 0.5568859853981479, + "cps_kids_of_reference_or_partner_positive": 0.558641425893528, + "cps_kids_of_reference_or_partner_under_18_positive": 0.2240125994897446, + "scf_employment_income": { + "p25": 23621.62162162162, + "p50": 71891.8918918919, + "p75": 159189.1891891892, + "p90": 259837.83783783787, + "mean": 130259.85946328501 + }, + "scf_household_income": { + "p25": 48270.27027027027, + "p50": 100648.64864864865, + "p75": 186918.9189189189, + "p90": 328648.64864864864, + "mean": 180220.35393913506 + }, + "cps_employment_income_household": { + "p25": 38272.0, + "p50": 82900.0, + "p75": 155388.67974777403, + "p90": 252016.93438084362, + "mean": 123965.18604683633 + }, + "cps_household_income_household": { + "p25": 64188.51669456113, + "p50": 122043.97988142398, + "p75": 203218.0, + "p90": 320036.5779984712, + "mean": 166351.77819530095 + }, + "cps_employment_income_spm_unit": { + "p25": 37000.0, + "p50": 81600.0, + "p75": 155000.0, + "p90": 249998.89957892447, + "mean": 122996.81407997927 + }, + "cps_household_income_spm_unit": { + "p25": 63153.59774069108, + "p50": 120900.0, + "p75": 202318.46309585773, + "p90": 317707.0663992214, + "mean": 165210.84922738944 + }, + "cps_employment_income_family": { + "p25": 36500.0, + "p50": 80009.1444996178, + "p75": 153014.52115728095, + "p90": 249998.89957892447, + "mean": 121749.8923112994 + }, + "cps_household_income_family": { + "p25": 61000.0, + "p50": 118409.84115188659, + "p75": 200442.52502611146, + "p90": 316000.0, + "mean": 163591.7473713767 + }, + "cps_employment_income_couple_and_children": { + "p25": 36200.0, + "p50": 79000.0, + "p75": 151471.3180734604, + "p90": 245013.16653888894, + "mean": 119309.41960271333 + }, + "cps_household_income_couple_and_children": { + "p25": 58967.75203674419, + "p50": 115648.27051673843, + "p75": 194475.84052737712, + "p90": 310684.71734198555, + "mean": 159538.9537478854 + }, + "cps_employment_income_couple": { + "p25": 30000.0, + "p50": 70000.0, + "p75": 134956.83826291453, + "p90": 219965.79340918665, + "mean": 108116.09074350675 + }, + "cps_household_income_couple": { + "p25": 49775.720983738975, + "p50": 99952.07665975994, + "p75": 174635.93590561362, + "p90": 290459.1090626288, + "mean": 145711.49579543713 + } + }, + "62_plus": { + "scf_married_or_partnered": 0.5111385359889865, + "cps_A_MARITL_1_2": 0.7388492319827763, + "cps_spouse_or_partner_present": 0.7433589060264927, + "scf_kids_positive": 0.15155608421924158, + "cps_kids_own_of_reference_positive": 0.23466102562126542, + "cps_kids_of_reference_or_partner_positive": 0.23611706304714625, + "cps_kids_of_reference_or_partner_under_18_positive": 0.0280574421521517, + "scf_employment_income": { + "p25": 0.0, + "p50": 0.0, + "p75": 33891.89189189189, + "p90": 103729.72972972973, + "mean": 40191.80953371061 + }, + "scf_household_income": { + "p25": 36972.97297297297, + "p50": 69837.83783783784, + "p75": 128378.37837837837, + "p90": 237243.24324324325, + "mean": 128923.04558037713 + }, + "cps_employment_income_household": { + "p25": 0.0, + "p50": 1970.5516301092812, + "p75": 49976.038329879964, + "p90": 141025.02027529074, + "mean": 49432.38346711528 + }, + "cps_household_income_household": { + "p25": 48581.68893862247, + "p50": 82502.01101963557, + "p75": 145234.83538180616, + "p90": 241173.30773902748, + "mean": 123362.53121783666 + }, + "cps_employment_income_spm_unit": { + "p25": 0.0, + "p50": 1697.5233922025736, + "p75": 49026.37489368272, + "p90": 139956.64890956884, + "mean": 48902.609141760426 + }, + "cps_household_income_spm_unit": { + "p25": 47930.98556183415, + "p50": 82060.05538651877, + "p75": 144032.70228142804, + "p90": 239693.7450230223, + "mean": 122592.85255142464 + }, + "cps_employment_income_family": { + "p25": 0.0, + "p50": 1500.0, + "p75": 47958.00352796082, + "p90": 137345.0744600264, + "mean": 48242.34563842785 + }, + "cps_household_income_family": { + "p25": 47509.73061402481, + "p50": 81383.00936982312, + "p75": 142350.9370965536, + "p90": 238470.92633795424, + "mean": 121850.6563805075 + }, + "cps_employment_income_couple_and_children": { + "p25": 0.0, + "p50": 403.6069603838287, + "p75": 45000.0, + "p90": 131400.0, + "mean": 46279.86395045464 + }, + "cps_household_income_couple_and_children": { + "p25": 46600.19389539599, + "p50": 79475.31977810201, + "p75": 137112.48037493764, + "p90": 233097.8623748759, + "mean": 119063.15046064938 + }, + "cps_employment_income_couple": { + "p25": 0.0, + "p50": 0.0, + "p75": 30033.106169737843, + "p90": 110000.0, + "mean": 39542.742825953 + }, + "cps_household_income_couple": { + "p25": 41528.197728847954, + "p50": 71904.81898503136, + "p75": 123930.40990673419, + "p90": 210593.9002452616, + "mean": 110218.86960885866 + } + } + } +} diff --git a/experiments/receipts/1073-predictor-shift-diagnosis.json b/experiments/receipts/1073-predictor-shift-diagnosis.json new file mode 100644 index 0000000000..45ecbed3db --- /dev/null +++ b/experiments/receipts/1073-predictor-shift-diagnosis.json @@ -0,0 +1,690 @@ +{ + "distributions": { + "18_24": { + "is_female": { + "scf": { + "share": 0.41798770703096133 + }, + "cps": { + "share": 0.45308552428744164 + } + }, + "cps_race": { + "scf": { + "1": 0.4055588573289111, + "2": 0.1494317074031604, + "3": 0.12691125859473804, + "4": 0.03117349448927348, + "7": 0.2869246821839171 + }, + "cps": { + "1": 0.4912125359470461, + "2": 0.1750171196976169, + "3": 0.2643223228596663, + "4": 0.04973560604712058, + "7": 0.01971241544855015 + } + }, + "is_married": { + "scf": { + "share": 0.388466732733208 + }, + "cps": { + "share": 0.24320062204068002 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.24031884047789556, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 2.0, + "p99": 3.0 + }, + "cps": { + "share_positive": 0.2522423862320045, + "p25": 0.0, + "p50": 0.0, + "p75": 1.0, + "p90": 1.0, + "p99": 3.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.8599811794959404, + "p25": 13351.351351351352, + "p50": 27729.72972972973, + "p75": 48270.27027027027, + "p90": 82162.16216216216, + "p99": 143783.7837837838 + }, + "cps": { + "share_positive": 0.9759394993350664, + "p25": 23040.0, + "p50": 46200.0, + "p75": 77991.10969769866, + "p90": 112891.24097794737, + "p99": 252966.59781704083 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.05424986155406579, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p99": 2464.864864864865 + }, + "cps": { + "share_positive": 0.28565123900592204, + "p25": 0.0, + "p50": 0.0, + "p75": 60.0, + "p90": 1709.816556616286, + "p99": 29054.057458670093 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.024318732528120387, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p99": 21567.567567567567 + }, + "cps": { + "share_positive": 0.18299893644132223, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 30971.512260709966, + "p99": 77291.65930125325 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9838529349150277, + "p25": 21567.567567567567, + "p50": 38000.0, + "p75": 64682.16216216216, + "p90": 87297.2972972973, + "p99": 146864.86486486485 + }, + "cps": { + "share_positive": 0.999999963347847, + "p25": 31220.185464984395, + "p50": 58862.3520972485, + "p75": 88793.53128444232, + "p90": 144160.0, + "p99": 375199.535233762 + } + } + }, + "25_34": { + "is_female": { + "scf": { + "share": 0.2619838404955461 + }, + "cps": { + "share": 0.30444635032847966 + } + }, + "cps_race": { + "scf": { + "1": 0.5017149524345604, + "2": 0.14942891698567948, + "3": 0.1917846516269202, + "4": 0.03776079752055218, + "7": 0.1193106814322877 + }, + "cps": { + "1": 0.5491809062140568, + "2": 0.13721256467989754, + "3": 0.21402341380294646, + "4": 0.06370355076664147, + "7": 0.03587956453645786 + } + }, + "is_married": { + "scf": { + "share": 0.5399894876417738 + }, + "cps": { + "share": 0.4522590121748441 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.3846077052668725, + "p25": 0.0, + "p50": 0.0, + "p75": 1.0, + "p90": 2.0, + "p99": 4.0 + }, + "cps": { + "share_positive": 0.4305778569704265, + "p25": 0.0, + "p50": 0.0, + "p75": 2.0, + "p90": 2.0, + "p99": 5.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.9253064946046667, + "p25": 36972.97297297297, + "p50": 71891.8918918919, + "p75": 114000.0, + "p90": 174594.5945945946, + "p99": 269081.08108108107 + }, + "cps": { + "share_positive": 0.9557187652328238, + "p25": 33000.0, + "p50": 62202.95507091948, + "p75": 113100.0, + "p90": 187083.69693085706, + "p99": 488000.0 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.1631292191129306, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 482.70270270270265, + "p99": 5751.351351351352 + }, + "cps": { + "share_positive": 0.4197204007983822, + "p25": 0.0, + "p50": 0.0, + "p75": 829.1927011444866, + "p90": 2750.923519968109, + "p99": 35128.73801214654 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.06155435215385703, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p99": 29783.783783783783 + }, + "cps": { + "share_positive": 0.182474707324782, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 28422.962257658946, + "p99": 118514.16659422085 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9989918938108396, + "p25": 46216.21621621621, + "p50": 82162.16216216216, + "p75": 125297.29729729731, + "p90": 184864.86486486485, + "p99": 437513.5135135135 + }, + "cps": { + "share_positive": 0.994540866676791, + "p25": 41073.922849105154, + "p50": 75949.0, + "p75": 133000.0, + "p90": 219819.68333705288, + "p99": 521387.8288052998 + } + } + }, + "35_49": { + "is_female": { + "scf": { + "share": 0.2530206375476159 + }, + "cps": { + "share": 0.21152846065969416 + } + }, + "cps_race": { + "scf": { + "1": 0.5370398830446967, + "2": 0.14963993063217354, + "3": 0.14916109018056492, + "4": 0.05640742673551139, + "7": 0.10775166940705336 + }, + "cps": { + "1": 0.5781912551424031, + "2": 0.13912110647748177, + "3": 0.18770513973830344, + "4": 0.0648546948157561, + "7": 0.03012780382605546 + } + }, + "is_married": { + "scf": { + "share": 0.6290734845229035 + }, + "cps": { + "share": 0.6653514555372676 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.6816213933375661, + "p25": 0.0, + "p50": 1.0, + "p75": 2.0, + "p90": 3.0, + "p99": 5.0 + }, + "cps": { + "share_positive": 0.7049748206449906, + "p25": 0.0, + "p50": 1.0, + "p75": 2.0, + "p90": 3.0, + "p99": 5.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.886081393441535, + "p25": 34918.91891891892, + "p50": 82162.16216216216, + "p75": 143783.7837837838, + "p90": 225945.94594594595, + "p99": 688108.1081081082 + }, + "cps": { + "share_positive": 0.9362493700847381, + "p25": 32644.68061928026, + "p50": 75973.07489577951, + "p75": 139956.64890956884, + "p90": 220559.3330568099, + "p99": 799260.4894895054 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.23097723402339967, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 2290.2702702702704, + "p99": 41081.08108108108 + }, + "cps": { + "share_positive": 0.43245290547981113, + "p25": 0.0, + "p50": 0.0, + "p75": 1200.2952505810445, + "p90": 5473.829683018923, + "p99": 102204.07752230708 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.11548337010251028, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 5135.135135135135, + "p99": 54432.43243243243 + }, + "cps": { + "share_positive": 0.19991842267775162, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 34432.23815943176, + "p99": 122184.8102383467 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9975323394286034, + "p25": 51351.35135135135, + "p50": 100648.64864864864, + "p75": 169459.45945945944, + "p90": 262364.3243243243, + "p99": 862702.7027027027 + }, + "cps": { + "share_positive": 0.9923842374071725, + "p25": 46409.00779119151, + "p50": 96035.69421902066, + "p75": 167809.5201741723, + "p90": 275503.0, + "p99": 971000.0 + } + } + }, + "50_61": { + "is_female": { + "scf": { + "share": 0.2735206442912563 + }, + "cps": { + "share": 0.18106391350030282 + } + }, + "cps_race": { + "scf": { + "1": 0.6150592830522598, + "2": 0.1537290366538993, + "3": 0.12412533592366588, + "4": 0.0410681313115236, + "7": 0.06601821305865142 + }, + "cps": { + "1": 0.6537814710133548, + "2": 0.11147038885346121, + "3": 0.15250900954288946, + "4": 0.06242020394741609, + "7": 0.019818926642878383 + } + }, + "is_married": { + "scf": { + "share": 0.5823377506882798 + }, + "cps": { + "share": 0.7105272865424017 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.4757664892740851, + "p25": 0.0, + "p50": 0.0, + "p75": 1.0, + "p90": 2.0, + "p99": 4.0 + }, + "cps": { + "share_positive": 0.5586414258935282, + "p25": 0.0, + "p50": 1.0, + "p75": 2.0, + "p90": 2.0, + "p99": 4.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.8120647798629679, + "p25": 23621.62162162162, + "p50": 71891.8918918919, + "p75": 159189.1891891892, + "p90": 259837.83783783787, + "p99": 1283783.7837837837 + }, + "cps": { + "share_positive": 0.9242217192397343, + "p25": 38272.0, + "p50": 82900.0, + "p75": 155388.67974777403, + "p90": 252016.93438084362, + "p99": 893039.7538139833 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.27101727557750777, + "p25": 0.0, + "p50": 0.0, + "p75": 102.70270270270271, + "p90": 7086.486486486487, + "p99": 114308.10810810812 + }, + "cps": { + "share_positive": 0.520315648054323, + "p25": 0.0, + "p50": 50.0, + "p75": 2393.0159509939276, + "p90": 12348.937448074455, + "p99": 177569.65127567103 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.24149444109701362, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 22594.594594594593, + "p99": 108864.86486486487 + }, + "cps": { + "share_positive": 0.32165954078256986, + "p25": 0.0, + "p50": 0.0, + "p75": 14103.402241167252, + "p90": 55600.57660509128, + "p99": 157776.23506719279 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9984637035763814, + "p25": 48270.27027027027, + "p50": 100648.64864864865, + "p75": 186918.9189189189, + "p90": 328648.64864864864, + "p99": 1582648.6486486485 + }, + "cps": { + "share_positive": 0.994115224234468, + "p25": 64188.51669456113, + "p50": 122043.97988142398, + "p75": 203218.0, + "p90": 320036.5779984712, + "p99": 1040938.4747342351 + } + } + }, + "62_plus": { + "is_female": { + "scf": { + "share": 0.3270495829974353 + }, + "cps": { + "share": 0.16284687085507255 + } + }, + "cps_race": { + "scf": { + "1": 0.7215070982985176, + "2": 0.11624912623121458, + "3": 0.05977273136960041, + "4": 0.03256850555294621, + "7": 0.06990253854772117 + }, + "cps": { + "1": 0.7797796558363141, + "2": 0.07935959144499927, + "3": 0.08314726942803088, + "4": 0.04345870240384795, + "7": 0.01425478088680773 + } + }, + "is_married": { + "scf": { + "share": 0.5111385359889865 + }, + "cps": { + "share": 0.7433589060264926 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.15155608421924158, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 1.0, + "p99": 2.0 + }, + "cps": { + "share_positive": 0.2361170630471462, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 1.0, + "p99": 3.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.3848493172407671, + "p25": 0.0, + "p50": 0.0, + "p75": 33891.89189189189, + "p90": 103729.72972972973, + "p99": 487837.83783783775 + }, + "cps": { + "share_positive": 0.5224639747168877, + "p25": 0.0, + "p50": 1970.5516301092812, + "p75": 49976.038329879964, + "p90": 141025.02027529074, + "p99": 600305.9996061828 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.360591087221194, + "p25": 0.0, + "p50": 0.0, + "p75": 1160.5405405405406, + "p90": 15405.405405405403, + "p99": 153437.83783783784 + }, + "cps": { + "share_positive": 0.5830926043365081, + "p25": 0.0, + "p50": 437.0202506310178, + "p75": 2891.2881910941755, + "p90": 19489.630485564376, + "p99": 161045.08384893282 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.8650097068004545, + "p25": 15405.405405405403, + "p50": 33891.89189189189, + "p75": 67783.78378378379, + "p90": 118108.10810810812, + "p99": 335837.8378378378 + }, + "cps": { + "share_positive": 0.7798703733427339, + "p25": 4362.970963141075, + "p50": 41057.42628607926, + "p75": 64539.98293667451, + "p90": 100348.95953445185, + "p99": 217029.24706127666 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9983582259004563, + "p25": 36972.97297297297, + "p50": 69837.83783783784, + "p75": 128378.37837837837, + "p90": 237243.24324324325, + "p99": 1087621.6216216215 + }, + "cps": { + "share_positive": 0.9845537936806925, + "p25": 48581.68893862247, + "p50": 82502.01101963557, + "p75": 145234.83538180616, + "p90": 241173.30773902748, + "p99": 897701.1335746899 + } + } + } + }, + "counterfactual_shares": { + "host_as_is": { + "all": 0.14160885581858304, + "18_24": 0.2079946693400335, + "25_34": 0.26288840763041604, + "35_49": 0.20667911833738084, + "50_61": 0.12965242867557655, + "62_plus": 0.0317293089245772 + }, + "map_employment_income": { + "all": 0.14550853383577372, + "18_24": 0.20821733454837177, + "25_34": 0.2822811879547042, + "35_49": 0.22214067557292733, + "50_61": 0.12263370638995597, + "62_plus": 0.026156416929239104 + }, + "map_interest_dividend_income": { + "all": 0.15826533773859558, + "18_24": 0.25113751499207576, + "25_34": 0.29426040903559375, + "35_49": 0.2253083018522811, + "50_61": 0.15493430900783606, + "62_plus": 0.031284222225362146 + }, + "map_social_security_pension_income": { + "all": 0.14681909389719208, + "18_24": 0.25129827692417706, + "25_34": 0.2717408539776387, + "35_49": 0.2205518487572771, + "50_61": 0.1255889841224885, + "62_plus": 0.030084654967087582 + }, + "map_household_income": { + "all": 0.1511521786657064, + "18_24": 0.2519469600079321, + "25_34": 0.29107702409361075, + "35_49": 0.22018752922954252, + "50_61": 0.13158606097402478, + "62_plus": 0.030002340966301628 + }, + "map_own_children_in_household": { + "all": 0.14533313698412845, + "18_24": 0.23320659588389747, + "25_34": 0.2648754761725435, + "35_49": 0.21431410469470719, + "50_61": 0.1348310653378586, + "62_plus": 0.03003456004297946 + }, + "map_all_continuous": { + "all": 0.17321348702539888, + "18_24": 0.30505039634570374, + "25_34": 0.34426263427236925, + "35_49": 0.2618932584423557, + "50_61": 0.14637659800134883, + "62_plus": 0.02301374608038495 + } + }, + "scf_observed_shares": { + "all": 0.18154956812200357, + "18_24": 0.22183897089190358, + "25_34": 0.3615365625657711, + "35_49": 0.27186430754586627, + "50_61": 0.1666515695303606, + "62_plus": 0.041870913769168856 + } +} diff --git a/experiments/receipts/1073-recipient-definition-variants.json b/experiments/receipts/1073-recipient-definition-variants.json new file mode 100644 index 0000000000..f7211fcf68 --- /dev/null +++ b/experiments/receipts/1073-recipient-definition-variants.json @@ -0,0 +1,1972 @@ +{ + "scf_2025": { + "share": 0.18154956812200357, + "parent_plus_share": 0.13750075307389342, + "parent_plus_by_reference_age": { + "24_and_younger": 0.0, + "25_to_34": 0.0036635524662026776, + "35_to_49": 0.041223882043333035, + "50_to_61": 0.352775474436351, + "62_and_older": 0.6050140082066444 + }, + "by_head_age_band": { + "24_and_younger": 0.22183897089190358, + "25_to_34": 0.3615365625657711, + "35_to_49": 0.27186430754586627, + "50_to_61": 0.1666515695303606, + "62_and_older": 0.041870913769168856 + }, + "by_income_quintile": { + "1": 0.1144757325207629, + "2": 0.1705804998024808, + "3": 0.17774016899539674, + "4": 0.26595688145561025, + "5": 0.1790121945626286 + } + }, + "variants": { + "v0_first_receipt": { + "definition": { + "reference_person": "oldest_adult", + "partnered": "married_spouse", + "children": "of_reference", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060217, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "The first receipt's construction.", + "share": 0.14412999386354491, + "parent_plus_share": 0.21495679635292878, + "parent_plus_by_reference_age": { + "24_and_younger": 0.05361481308447381, + "25_to_34": 0.00022032123488229902, + "35_to_49": 0.03526511572758998, + "50_to_61": 0.4335550013453078, + "62_and_older": 0.7000193215077479 + }, + "by_head_age_band": { + "24_and_younger": 0.2267174320421409, + "25_to_34": 0.2669429791293293, + "35_to_49": 0.22339688829964577, + "50_to_61": 0.14218578094851128, + "62_and_older": 0.03655929815086838 + }, + "by_income_quintile": { + "1": 0.12721563759800417, + "2": 0.1358771691149805, + "3": 0.13382646407799811, + "4": 0.19993030155269514, + "5": 0.12378308052696536 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7419574258458654, + "mean_among_borrowers_error_pct": 8.357217142313411, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.061856909978440955, + "head_age_band_share_mae_pp": 3.5543373606609885, + "race_share_mae_pp": 2.8343426729118955, + "income_quintile_share_mae_pp": 4.2522526924143715, + "borrower_balance_band_tvd": 0.06971797938468582, + "borrower_age_band_tvd": 0.08466988479278444 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14412999386354491, + "mean_among_borrowers": 44483.09961323457, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 785628046119.6729, + "parent_plus_share_of_total": 0.21495679635292878 + }, + "household_balance_bands": { + "under_5k": 0.09597901426395242, + "5k_to_10k": 0.1620676904114682, + "10k_to_20k": 0.15178493618645325, + "20k_to_40k": 0.22796648343692355, + "40k_to_60k": 0.09915026443778263, + "60k_to_80k": 0.10849192930212849, + "80k_to_100k": 0.056043492646084624, + "100k_to_200k": 0.06743544596300373, + "200k_plus": 0.031080743352203163 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.2267174320421409, + "mean_among_borrowers": 18670.63953159211, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2669429791293293, + "mean_among_borrowers": 36447.86184035785, + "median_among_borrowers": 20200.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22339688829964577, + "mean_among_borrowers": 44315.447461546944, + "median_among_borrowers": 25670.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14218578094851128, + "mean_among_borrowers": 49552.354866035355, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03655929815086838, + "mean_among_borrowers": 65818.86253190483, + "median_among_borrowers": 38000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13333034770117472, + "mean_among_borrowers": 42577.59434933696, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902667788102, + "mean_among_borrowers": 47993.699863516216, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12291729609744638, + "mean_among_borrowers": 46285.94561484015, + "median_among_borrowers": 24764.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15337638725400463, + "mean_among_borrowers": 47893.571558628166, + "median_among_borrowers": 24000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12721563759800417, + "mean_among_borrowers": 38171.39283093021, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1358771691149805, + "mean_among_borrowers": 45461.58729969715, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13382646407799811, + "mean_among_borrowers": 40922.60303950533, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19993030155269514, + "mean_among_borrowers": 45004.488576349744, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12378308052696536, + "mean_among_borrowers": 52902.03665110157, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18461703.3923496, + "mean_balance": 42554.4723270352, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.1039144177980828, + "5k_to_10k": 0.16665917220324747, + "10k_to_20k": 0.15515901493993045, + "20k_to_40k": 0.23029463650667883, + "40k_to_60k": 0.10426754245310213, + "60k_to_80k": 0.09578243033185131, + "80k_to_100k": 0.05074417014558102, + "100k_to_200k": 0.06376833884555547, + "200k_plus": 0.029410276775970686 + }, + "age_bands": { + "24_and_younger": 0.03769253987750687, + "25_to_34": 0.2662405413523508, + "35_to_49": 0.39817213354338704, + "50_to_61": 0.21071813444930676, + "62_and_older": 0.08717665077744856 + }, + "graduate_share_of_own_original_principal": 0.5360724443816975, + "ibr_new_borrower_share": 0.605734686749079 + } + } + }, + "v1_partner_counts_as_married": { + "definition": { + "reference_person": "oldest_adult", + "partnered": "spouse_or_partner", + "children": "of_reference", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060217, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "SCF MARRIED is married or living with a partner (X8023 in 1-2); A_MARITL 1-2 drops cohabiting partners.", + "share": 0.14421786795177846, + "parent_plus_share": 0.23835459647602736, + "parent_plus_by_reference_age": { + "24_and_younger": 2.253143474595712e-05, + "25_to_34": 0.0031437172991370397, + "35_to_49": 0.033462461180418936, + "50_to_61": 0.5087392931361828, + "62_and_older": 0.7514633083579594 + }, + "by_head_age_band": { + "24_and_younger": 0.22671795358234773, + "25_to_34": 0.2681773827190304, + "35_to_49": 0.22322300770469652, + "50_to_61": 0.14171355085128, + "62_and_older": 0.03672435234678404 + }, + "by_income_quintile": { + "1": 0.1272156431154305, + "2": 0.1359920694211529, + "3": 0.13490203218158234, + "4": 0.19906487838581546, + "5": 0.12389784553658897 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.733170017022511, + "mean_among_borrowers_error_pct": 8.963130475592095, + "median_among_borrowers_error_pct": 1.200000000000001, + "household_balance_band_tvd": 0.06098877733974077, + "head_age_band_share_mae_pp": 3.5392808495963997, + "race_share_mae_pp": 2.895639943104994, + "income_quintile_share_mae_pp": 4.2434565977128855, + "borrower_balance_band_tvd": 0.06591147421877885, + "borrower_age_band_tvd": 0.07517303040860317 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14421786795177846, + "mean_among_borrowers": 44731.84080345749, + "median_among_borrowers": 25300.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 790502796606.3756, + "parent_plus_share_of_total": 0.23835459647602736 + }, + "household_balance_bands": { + "under_5k": 0.09724840853122779, + "5k_to_10k": 0.15964535820912895, + "10k_to_20k": 0.14731451015137764, + "20k_to_40k": 0.23069297158741, + "40k_to_60k": 0.10134840208870029, + "60k_to_80k": 0.10851424309963001, + "80k_to_100k": 0.05607431201281855, + "100k_to_200k": 0.06747102087873265, + "200k_plus": 0.03169077344097403 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.22671795358234773, + "mean_among_borrowers": 18861.163422392947, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2681773827190304, + "mean_among_borrowers": 37146.8559849801, + "median_among_borrowers": 22000.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22322300770469652, + "mean_among_borrowers": 44393.29221805572, + "median_among_borrowers": 25944.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14171355085128, + "mean_among_borrowers": 49673.12225637845, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03672435234678404, + "mean_among_borrowers": 65867.18117871678, + "median_among_borrowers": 39000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13395046003400674, + "mean_among_borrowers": 42718.245961511835, + "median_among_borrowers": 25300.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902879591022, + "mean_among_borrowers": 48460.870926229596, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12189634611539013, + "mean_among_borrowers": 46674.41390252431, + "median_among_borrowers": 25000.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15132533197747572, + "mean_among_borrowers": 48377.79858657893, + "median_among_borrowers": 24700.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.1272156431154305, + "mean_among_borrowers": 38691.57607287244, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1359920694211529, + "mean_among_borrowers": 45423.29084841887, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13490203218158234, + "mean_among_borrowers": 41268.331115616085, + "median_among_borrowers": 25300.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19906487838581546, + "mean_among_borrowers": 45324.769240708556, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12389784553658897, + "mean_among_borrowers": 52992.43425311495, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18499654.177020565, + "mean_balance": 42730.679667963865, + "median_balance": 24321.0, + "balance_bands": { + "under_5k": 0.10864552035546077, + "5k_to_10k": 0.16261363075134405, + "10k_to_20k": 0.155785250662947, + "20k_to_40k": 0.22691872382328185, + "40k_to_60k": 0.10271670933861446, + "60k_to_80k": 0.10059174421550436, + "80k_to_100k": 0.04980040443960724, + "100k_to_200k": 0.06284484580513604, + "200k_plus": 0.03008317060810413 + }, + "age_bands": { + "24_and_younger": 0.039091157227139205, + "25_to_34": 0.27433877838689963, + "35_to_49": 0.3897180634972089, + "50_to_61": 0.20575528706790563, + "62_and_older": 0.09109671382084646 + }, + "graduate_share_of_own_original_principal": 0.5381213307355639, + "ibr_new_borrower_share": 0.6057402495033247 + } + } + }, + "v2_children_of_reference_or_partner": { + "definition": { + "reference_person": "oldest_adult", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060217, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "SCF KIDS counts children of the reference person or spouse/partner, any age, anywhere in the household listing.", + "share": 0.1394955127190334, + "parent_plus_share": 0.17976252753958277, + "parent_plus_by_reference_age": { + "24_and_younger": 1.8950713199477208e-09, + "25_to_34": 0.003974829123535003, + "35_to_49": 0.023750250056890892, + "50_to_61": 0.4267091997855419, + "62_and_older": 0.6189997224359128 + }, + "by_head_age_band": { + "24_and_younger": 0.23591005978245044, + "25_to_34": 0.2765552999001228, + "35_to_49": 0.21374384005656138, + "50_to_61": 0.13109688553788776, + "62_and_older": 0.03292444110763649 + }, + "by_income_quintile": { + "1": 0.12259529843027182, + "2": 0.13647608213109577, + "3": 0.1390432941963818, + "4": 0.1883195761175596, + "5": 0.11102922220126195 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -4.205405540297017, + "mean_among_borrowers_error_pct": 5.248358030394584, + "median_among_borrowers_error_pct": 1.200000000000001, + "household_balance_band_tvd": 0.04583400383836792, + "head_age_band_share_mae_pp": 4.033479513990105, + "race_share_mae_pp": 3.346280982970381, + "income_quintile_share_mae_pp": 4.5308227215865235, + "borrower_balance_band_tvd": 0.05793805362184862, + "borrower_age_band_tvd": 0.05544035124223032 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.1394955127190334, + "mean_among_borrowers": 43206.84231163398, + "median_among_borrowers": 25300.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 738550734853.6542, + "parent_plus_share_of_total": 0.17976252753958277 + }, + "household_balance_bands": { + "under_5k": 0.09939228853825689, + "5k_to_10k": 0.1535814556424139, + "10k_to_20k": 0.15338979008804154, + "20k_to_40k": 0.23417095609696748, + "40k_to_60k": 0.10944941047868599, + "60k_to_80k": 0.10279164254586112, + "80k_to_100k": 0.04997604706138593, + "100k_to_200k": 0.06911469871793441, + "200k_plus": 0.028133710830452702 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.23591005978245044, + "mean_among_borrowers": 21290.46116231051, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2765552999001228, + "mean_among_borrowers": 36585.71819351528, + "median_among_borrowers": 21600.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.21374384005656138, + "mean_among_borrowers": 44071.0450196076, + "median_among_borrowers": 26500.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.13109688553788776, + "mean_among_borrowers": 49220.30130728327, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03292444110763649, + "mean_among_borrowers": 55939.813048540644, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.1289552997263875, + "mean_among_borrowers": 41158.906258343646, + "median_among_borrowers": 25300.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22416810893631395, + "mean_among_borrowers": 47150.30266408629, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.11808625225094552, + "mean_among_borrowers": 44282.98610623642, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.14618586441452036, + "mean_among_borrowers": 47779.9480237213, + "median_among_borrowers": 24700.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12259529843027182, + "mean_among_borrowers": 38343.347009388315, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.13647608213109577, + "mean_among_borrowers": 45292.80429539413, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.1390432941963818, + "mean_among_borrowers": 38379.766451262985, + "median_among_borrowers": 25000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.1883195761175596, + "mean_among_borrowers": 43996.9806856208, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.11102922220126195, + "mean_among_borrowers": 50716.140548307994, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 17825507.124679778, + "mean_balance": 41432.242554890094, + "median_balance": 25000.0, + "balance_bands": { + "under_5k": 0.10587106866451161, + "5k_to_10k": 0.15716435659164057, + "10k_to_20k": 0.15459448173794518, + "20k_to_40k": 0.23795597452795925, + "40k_to_60k": 0.11465535055149564, + "60k_to_80k": 0.09453438820382777, + "80k_to_100k": 0.043584550860481554, + "100k_to_200k": 0.06582616567410109, + "200k_plus": 0.025813663188037198 + }, + "age_bands": { + "24_and_younger": 0.04174644796694323, + "25_to_34": 0.2914161668134685, + "35_to_49": 0.383747423990049, + "50_to_61": 0.1994666870240408, + "62_and_older": 0.08362327420549835 + }, + "graduate_share_of_own_original_principal": 0.5133189129620963, + "ibr_new_borrower_share": 0.5894419667702057 + } + } + }, + "v3_householder_couple_reference_person": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060217, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "The SCF reference person comes from the economically dominant individual or couple (the man, or the older partner); the CPS householder anchors that unit, where the oldest adult can be a co-resident parent or roommate.", + "share": 0.14160885581858304, + "parent_plus_share": 0.16635099149589067, + "parent_plus_by_reference_age": { + "24_and_younger": 0.00017157399333482214, + "25_to_34": 0.005150069210217926, + "35_to_49": 0.02057959141870075, + "50_to_61": 0.42901843742863854, + "62_and_older": 0.6324187709904693 + }, + "by_head_age_band": { + "24_and_younger": 0.19846806978919576, + "25_to_34": 0.26288840763041604, + "35_to_49": 0.20667911833738084, + "50_to_61": 0.12965242867557655, + "62_and_older": 0.0317293089245772 + }, + "by_income_quintile": { + "1": 0.1262566286047617, + "2": 0.14177829750298432, + "3": 0.1357961209373581, + "4": 0.19272855509869874, + "5": 0.11146869978027 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.994071230342053, + "mean_among_borrowers_error_pct": 3.531511434829504, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.03457855760861366, + "head_age_band_share_mae_pp": 4.686899818918479, + "race_share_mae_pp": 2.880989091350316, + "income_quintile_share_mae_pp": 4.46597935161608, + "borrower_balance_band_tvd": 0.048461501202019044, + "borrower_age_band_tvd": 0.027331980297340144 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14160885581858304, + "mean_among_borrowers": 42502.03777581002, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 737509698707.2527, + "parent_plus_share_of_total": 0.16635099149589067 + }, + "household_balance_bands": { + "under_5k": 0.09993330421288438, + "5k_to_10k": 0.1508675449396742, + "10k_to_20k": 0.16976059764222085, + "20k_to_40k": 0.2275283363657654, + "40k_to_60k": 0.11015794699391307, + "60k_to_80k": 0.09824782711371129, + "80k_to_100k": 0.046130984805146207, + "100k_to_200k": 0.0690932668947163, + "200k_plus": 0.028280191031968407 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.19846806978919576, + "mean_among_borrowers": 19571.066784291823, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.26288840763041604, + "mean_among_borrowers": 38450.03383676039, + "median_among_borrowers": 21000.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.20667911833738084, + "mean_among_borrowers": 42931.01574498944, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.12965242867557655, + "mean_among_borrowers": 47692.45531862902, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.0317293089245772, + "mean_among_borrowers": 58604.73232987876, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.12977996862923966, + "mean_among_borrowers": 41370.06905388904, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.2294764658429962, + "mean_among_borrowers": 45845.065773264745, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11656658232690145, + "mean_among_borrowers": 42058.17041201836, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.16018418419383262, + "mean_among_borrowers": 43738.317358090426, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.1262566286047617, + "mean_among_borrowers": 38482.73985057685, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.14177829750298432, + "mean_among_borrowers": 42757.013143750206, + "median_among_borrowers": 25000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.1357961209373581, + "mean_among_borrowers": 36845.744104474776, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19272855509869874, + "mean_among_borrowers": 42892.848389119885, + "median_among_borrowers": 25629.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.11146869978027, + "mean_among_borrowers": 52943.922322276216, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18148489.81162718, + "mean_balance": 40637.52446414318, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.11021695554879672, + "5k_to_10k": 0.152438368597162, + "10k_to_20k": 0.16997945471178874, + "20k_to_40k": 0.23626211817475332, + "40k_to_60k": 0.10727311904109983, + "60k_to_80k": 0.09305715931666757, + "80k_to_100k": 0.03975383525974363, + "100k_to_200k": 0.06463705675017781, + "200k_plus": 0.026381932599810472 + }, + "age_bands": { + "24_and_younger": 0.05449396163740877, + "25_to_34": 0.30902579405669556, + "35_to_49": 0.3785313140100526, + "50_to_61": 0.18776495318129746, + "62_and_older": 0.07018397711454578 + }, + "graduate_share_of_own_original_principal": 0.519651843154687, + "ibr_new_borrower_share": 0.609045516567325 + } + } + }, + "v4a_spm_unit_income": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "spm_unit" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060217, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "SCF income covers the primary economic unit; the SPM unit is the CPS resource-sharing unit.", + "share": 0.1422480075434781, + "parent_plus_share": 0.17924572328488167, + "parent_plus_by_reference_age": { + "24_and_younger": 2.0364004798094018e-08, + "25_to_34": 0.0035008167958351234, + "35_to_49": 0.02945856222174237, + "50_to_61": 0.47869069913423223, + "62_and_older": 0.6254994464664675 + }, + "by_head_age_band": { + "24_and_younger": 0.19738286662117155, + "25_to_34": 0.26409298361810984, + "35_to_49": 0.2083709419645182, + "50_to_61": 0.1302801328964551, + "62_and_older": 0.031441439273676396 + }, + "by_income_quintile": { + "1": 0.12941404718889285, + "2": 0.14024648681888457, + "3": 0.13552164034883177, + "4": 0.193981131880552, + "5": 0.11208227675067776 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.930156057852546, + "mean_among_borrowers_error_pct": 3.5620052430742177, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.03742705681897808, + "head_age_band_share_mae_pp": 4.643879198582787, + "race_share_mae_pp": 2.863735982595237, + "income_quintile_share_mae_pp": 4.527930473706005, + "borrower_balance_band_tvd": 0.039884034926986736, + "borrower_age_band_tvd": 0.035403180196326975 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.1422480075434781, + "mean_among_borrowers": 42514.55617694204, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 741056653493.6942, + "parent_plus_share_of_total": 0.17924572328488167 + }, + "household_balance_bands": { + "under_5k": 0.09952116824675251, + "5k_to_10k": 0.15435171448733467, + "10k_to_20k": 0.16562804147553756, + "20k_to_40k": 0.22674020689102248, + "40k_to_60k": 0.11279034359745732, + "60k_to_80k": 0.09923618460456847, + "80k_to_100k": 0.04557077463266375, + "100k_to_200k": 0.0678800899029161, + "200k_plus": 0.028281476161747247 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.19738286662117155, + "mean_among_borrowers": 19442.015941146798, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.26409298361810984, + "mean_among_borrowers": 38436.58028829374, + "median_among_borrowers": 20500.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.2083709419645182, + "mean_among_borrowers": 42853.01799862837, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.1302801328964551, + "mean_among_borrowers": 47861.768338455506, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.031441439273676396, + "mean_among_borrowers": 58939.872552347515, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.13097596521508803, + "mean_among_borrowers": 41202.78031031787, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.22831695407527614, + "mean_among_borrowers": 45683.93405244018, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11574381829957685, + "mean_among_borrowers": 41667.139357186345, + "median_among_borrowers": 22000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.16166058775323208, + "mean_among_borrowers": 45928.990341343546, + "median_among_borrowers": 25000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24498252.26087785, + "share_with_federal_education_debt": 0.12941404718889285, + "mean_among_borrowers": 38621.5541253665, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24516086.12746574, + "share_with_federal_education_debt": 0.14024648681888457, + "mean_among_borrowers": 43623.80714350449, + "median_among_borrowers": 26087.0 + }, + "3": { + "household_weight": 24508573.098401047, + "share_with_federal_education_debt": 0.13552164034883177, + "mean_among_borrowers": 35795.30093902187, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 24504387.547067694, + "share_with_federal_education_debt": 0.193981131880552, + "mean_among_borrowers": 41327.58298000465, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24509792.851870958, + "share_with_federal_education_debt": 0.11208227675067776, + "mean_among_borrowers": 55796.96154188077, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18198229.29909551, + "mean_balance": 40721.360376007906, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.10990097377367777, + "5k_to_10k": 0.15597631321107155, + "10k_to_20k": 0.1732232936496721, + "20k_to_40k": 0.22454924901870585, + "40k_to_60k": 0.11005065222546452, + "60k_to_80k": 0.09520726630411006, + "80k_to_100k": 0.04158394034849486, + "100k_to_200k": 0.0635499527510756, + "200k_plus": 0.025958358717727704 + }, + "age_bands": { + "24_and_younger": 0.0529887481725192, + "25_to_34": 0.3002110376537959, + "35_to_49": 0.3829114048942107, + "50_to_61": 0.19230144811764502, + "62_and_older": 0.07158736116182915 + }, + "graduate_share_of_own_original_principal": 0.536561414303497, + "ibr_new_borrower_share": 0.6249721194428659 + } + } + }, + "v4b_couple_and_children_income": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060217, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "The PEU narrowed to the couple and their children.", + "share": 0.14453729401731658, + "parent_plus_share": 0.15519605817424037, + "parent_plus_by_reference_age": { + "24_and_younger": 1.352905945393419e-08, + "25_to_34": 0.00013998491463271278, + "35_to_49": 0.019468440588935952, + "50_to_61": 0.39136120955142256, + "62_and_older": 0.5990857762189746 + }, + "by_head_age_band": { + "24_and_younger": 0.2109485663269708, + "25_to_34": 0.2709152074679265, + "35_to_49": 0.20910334189997756, + "50_to_61": 0.1331061797690564, + "62_and_older": 0.03120975131828736 + }, + "by_income_quintile": { + "1": 0.13612388616760965, + "2": 0.1459793402381904, + "3": 0.13033686972701333, + "4": 0.1919836615881704, + "5": 0.11826299055685831 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7012274104686993, + "mean_among_borrowers_error_pct": 2.7320841095266557, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.040156439181474324, + "head_age_band_share_mae_pp": 4.169585550417035, + "race_share_mae_pp": 2.569878492307957, + "income_quintile_share_mae_pp": 4.567500727054615, + "borrower_balance_band_tvd": 0.04019409665288553, + "borrower_age_band_tvd": 0.03235991026200852 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14453729401731658, + "mean_among_borrowers": 42173.85469504409, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 746948718213.9073, + "parent_plus_share_of_total": 0.15519605817424037 + }, + "household_balance_bands": { + "under_5k": 0.10775015906187184, + "5k_to_10k": 0.1528994221663368, + "10k_to_20k": 0.15878382717932804, + "20k_to_40k": 0.23707174823778418, + "40k_to_60k": 0.11254827709408233, + "60k_to_80k": 0.09506114425698398, + "80k_to_100k": 0.044996423198641146, + "100k_to_200k": 0.05939892961997767, + "200k_plus": 0.03149006918499395 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.2109485663269708, + "mean_among_borrowers": 22164.502413760794, + "median_among_borrowers": 14000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.2709152074679265, + "mean_among_borrowers": 36647.47528338518, + "median_among_borrowers": 21600.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.20910334189997756, + "mean_among_borrowers": 42586.38683331958, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.1331061797690564, + "mean_among_borrowers": 49336.50631455528, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.03120975131828736, + "mean_among_borrowers": 58478.07907213402, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.13312784049029713, + "mean_among_borrowers": 40718.755628997955, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.22788477777157826, + "mean_among_borrowers": 47824.23980122162, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11755065774394279, + "mean_among_borrowers": 39599.36261332915, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.1698883489488461, + "mean_among_borrowers": 44027.69290376679, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.13612388616760965, + "mean_among_borrowers": 38386.48367756073, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.1459793402381904, + "mean_among_borrowers": 42114.80663102074, + "median_among_borrowers": 23000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.13033686972701333, + "mean_among_borrowers": 39083.07983863153, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.1919836615881704, + "mean_among_borrowers": 42068.527873712716, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.11826299055685831, + "mean_among_borrowers": 50185.22911179778, + "median_among_borrowers": 26814.0 + } + }, + "borrowers": { + "borrower_weight": 18531371.849430196, + "mean_balance": 40307.25432973677, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.11848284445361304, + "5k_to_10k": 0.15391028318710992, + "10k_to_20k": 0.16124932842055162, + "20k_to_40k": 0.23782638366100617, + "40k_to_60k": 0.11453284638670623, + "60k_to_80k": 0.09065262966356538, + "80k_to_100k": 0.0390975463353386, + "100k_to_200k": 0.05475398408033307, + "200k_plus": 0.029494153811776037 + }, + "age_bands": { + "24_and_younger": 0.05515156793888616, + "25_to_34": 0.3089032338721271, + "35_to_49": 0.3729683379284752, + "50_to_61": 0.19389979448680775, + "62_and_older": 0.06907706577370382 + }, + "graduate_share_of_own_original_principal": 0.497501055409478, + "ibr_new_borrower_share": 0.6165370510095016 + } + } + }, + "r1_couple_income_drop_interest_dividends": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17429665336106148, + "parent_plus_share": 0.12799128561614068, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.7252914760942086, + "mean_among_borrowers_error_pct": -6.467428746293713, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03359502345865293, + "head_age_band_share_mae_pp": 1.1373174155641717, + "race_share_mae_pp": 1.2886634772442556, + "income_quintile_share_mae_pp": 0.8724139723993973, + "borrower_balance_band_tvd": 0.03705671769663749, + "borrower_age_band_tvd": 0.013839814547848696 + } + }, + "reason": "CPS interest and dividends are PUF-imputed at tax-unit level; at reference ages 25-34, 42% of host households have some against 16% of SCF families, so equal values do not mean equal positions.", + "share": 0.16068368510572498, + "parent_plus_share": 0.16139644121762028, + "parent_plus_by_reference_age": { + "24_and_younger": 4.3569134442118664e-10, + "25_to_34": 0.0006982323042478812, + "35_to_49": 0.01612499187177073, + "50_to_61": 0.3707882978015445, + "62_and_older": 0.7327689853082329 + }, + "by_head_age_band": { + "24_and_younger": 0.22636975285526706, + "25_to_34": 0.28177775577348513, + "35_to_49": 0.23675283607454572, + "50_to_61": 0.16663378760977693, + "62_and_older": 0.029689656574013047 + }, + "by_income_quintile": { + "1": 0.14148727103894412, + "2": 0.14782938572653392, + "3": 0.14763888154405314, + "4": 0.2260334704473883, + "5": 0.1404236189948884 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -2.086588301627859, + "mean_among_borrowers_error_pct": 5.419007269798315, + "median_among_borrowers_error_pct": -1.200000000000001, + "household_balance_band_tvd": 0.03897139087869254, + "head_age_band_share_mae_pp": 2.6320019868541893, + "race_share_mae_pp": 1.715677598047579, + "income_quintile_share_mae_pp": 3.1675185324286774, + "borrower_balance_band_tvd": 0.03809557472261625, + "borrower_age_band_tvd": 0.05104758904754849 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16068368510572498, + "mean_among_borrowers": 43276.897701717964, + "median_among_borrowers": 24700.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 852109629770.2631, + "parent_plus_share_of_total": 0.16139644121762028 + }, + "household_balance_bands": { + "under_5k": 0.10449852122435789, + "5k_to_10k": 0.15065860397660183, + "10k_to_20k": 0.16882661644138566, + "20k_to_40k": 0.23081137027456972, + "40k_to_60k": 0.10882288460069202, + "60k_to_80k": 0.09579589273732184, + "80k_to_100k": 0.04072416122031918, + "100k_to_200k": 0.06714054669721696, + "200k_plus": 0.03272140282753487 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.22636975285526706, + "mean_among_borrowers": 20232.49190929045, + "median_among_borrowers": 7940.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28177775577348513, + "mean_among_borrowers": 35902.8761787989, + "median_among_borrowers": 22000.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.23675283607454572, + "mean_among_borrowers": 45150.70392260933, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16663378760977693, + "mean_among_borrowers": 50437.639301615825, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.029689656574013047, + "mean_among_borrowers": 58758.63181213605, + "median_among_borrowers": 40000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.14966927917838, + "mean_among_borrowers": 42682.42788830105, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.25697435771393307, + "mean_among_borrowers": 46819.536501712115, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.12815612550195352, + "mean_among_borrowers": 36458.037818012526, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.1749032891253265, + "mean_among_borrowers": 49540.69039492447, + "median_among_borrowers": 21000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14148727103894412, + "mean_among_borrowers": 42475.669637712905, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.14782938572653392, + "mean_among_borrowers": 41544.88128808425, + "median_among_borrowers": 21997.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14763888154405314, + "mean_among_borrowers": 39417.16786016642, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.2260334704473883, + "mean_among_borrowers": 41212.809485576174, + "median_among_borrowers": 27000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1404236189948884, + "mean_among_borrowers": 53288.26313186684, + "median_among_borrowers": 25000.0 + } + }, + "borrowers": { + "borrower_weight": 20417273.566150248, + "mean_balance": 41734.74127235938, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.11809168217956414, + "5k_to_10k": 0.15130000841757468, + "10k_to_20k": 0.16989131376762578, + "20k_to_40k": 0.2286785190554563, + "40k_to_60k": 0.11099082001348541, + "60k_to_80k": 0.08837266814620766, + "80k_to_100k": 0.03787968228128524, + "100k_to_200k": 0.06370360539613464, + "200k_plus": 0.031091700742666088 + }, + "age_bands": { + "24_and_younger": 0.05398123485212966, + "25_to_34": 0.29133756126405175, + "35_to_49": 0.3828432273021299, + "50_to_61": 0.2105771421991523, + "62_and_older": 0.06126083438253632 + }, + "graduate_share_of_own_original_principal": 0.4992132097267167, + "ibr_new_borrower_share": 0.5832347045593139 + } + } + }, + "r2_couple_income_drop_interest_dividends_and_ss_pension": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "share": 0.1785134113597766, + "parent_plus_share": 0.12228737578012734, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.30361567622269603, + "mean_among_borrowers_error_pct": -7.119030826113226, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03158207267342461, + "head_age_band_share_mae_pp": 1.2596405967540016, + "race_share_mae_pp": 1.069077118976515, + "income_quintile_share_mae_pp": 0.6750791601254419, + "borrower_balance_band_tvd": 0.032948871945896414, + "borrower_age_band_tvd": 0.014803476410333236 + } + }, + "reason": "As r1, and Social Security or pension income: 18% of host households at 25-34 have some against 6% of SCF families.", + "share": 0.16413671000206656, + "parent_plus_share": 0.16758194869072, + "parent_plus_by_reference_age": { + "24_and_younger": 0.0009290910285421434, + "25_to_34": 0.003457206986791982, + "35_to_49": 0.020444291263711122, + "50_to_61": 0.3174101032569371, + "62_and_older": 0.7753235949011145 + }, + "by_head_age_band": { + "24_and_younger": 0.21797195752357604, + "25_to_34": 0.28107974022687005, + "35_to_49": 0.24429588401566177, + "50_to_61": 0.16352578267462928, + "62_and_older": 0.037347739302272136 + }, + "by_income_quintile": { + "1": 0.14257834053880739, + "2": 0.15791199105287693, + "3": 0.14719109419650156, + "4": 0.23072142160681058, + "5": 0.1422795004455311 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.7412858119937007, + "mean_among_borrowers_error_pct": 3.965667719599386, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.046105502753231414, + "head_age_band_share_mae_pp": 2.3908244112012222, + "race_share_mae_pp": 0.9415277758683394, + "income_quintile_share_mae_pp": 2.865766910648814, + "borrower_balance_band_tvd": 0.04473225112436991, + "borrower_age_band_tvd": 0.049566199767822304 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16413671000206656, + "mean_among_borrowers": 42680.26879514089, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 858421208954.9308, + "parent_plus_share_of_total": 0.16758194869072 + }, + "household_balance_bands": { + "under_5k": 0.1179407532730876, + "5k_to_10k": 0.1324473758996882, + "10k_to_20k": 0.1827494270561508, + "20k_to_40k": 0.23417620772637385, + "40k_to_60k": 0.10433535383061796, + "60k_to_80k": 0.0976346772652531, + "80k_to_100k": 0.0379014108379451, + "100k_to_200k": 0.05781213394341271, + "200k_plus": 0.03500266016747067 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.21797195752357604, + "mean_among_borrowers": 20475.641978208863, + "median_among_borrowers": 12662.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28107974022687005, + "mean_among_borrowers": 34573.95711939027, + "median_among_borrowers": 20100.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.24429588401566177, + "mean_among_borrowers": 44478.05328490035, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16352578267462928, + "mean_among_borrowers": 48394.24067058906, + "median_among_borrowers": 28742.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.037347739302272136, + "mean_among_borrowers": 61031.6733341336, + "median_among_borrowers": 34000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15291016385941802, + "mean_among_borrowers": 42266.19101485241, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.24861685783459786, + "mean_among_borrowers": 45500.64480289935, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.13835157436352993, + "mean_among_borrowers": 37031.934121143146, + "median_among_borrowers": 19200.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.18407544859054642, + "mean_among_borrowers": 48161.13603160578, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14257834053880739, + "mean_among_borrowers": 43328.84042017116, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.15791199105287693, + "mean_among_borrowers": 39522.35712436794, + "median_among_borrowers": 20000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14719109419650156, + "mean_among_borrowers": 38670.33311547162, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.23072142160681058, + "mean_among_borrowers": 41126.05793612391, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1422795004455311, + "mean_among_borrowers": 52202.66543686702, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 21223403.433088087, + "mean_balance": 40446.915673129966, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.12533388559616318, + "5k_to_10k": 0.13601201669515878, + "10k_to_20k": 0.18997031373843593, + "20k_to_40k": 0.2351308957545589, + "40k_to_60k": 0.10824433293789903, + "60k_to_80k": 0.08668533742450127, + "80k_to_100k": 0.03163070469899026, + "100k_to_200k": 0.05604778033785523, + "200k_plus": 0.03094473281643728 + }, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257 + }, + "graduate_share_of_own_original_principal": 0.4826934460430417, + "ibr_new_borrower_share": 0.6147006160485154 + } + } + }, + "r3_household_income_drop_interest_dividends_and_ss_pension": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "share": 0.1785134113597766, + "parent_plus_share": 0.12228737578012734, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.30361567622269603, + "mean_among_borrowers_error_pct": -7.119030826113226, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03158207267342461, + "head_age_band_share_mae_pp": 1.2596405967540016, + "race_share_mae_pp": 1.069077118976515, + "income_quintile_share_mae_pp": 0.6750791601254419, + "borrower_balance_band_tvd": 0.032948871945896414, + "borrower_age_band_tvd": 0.014803476410333236 + } + }, + "reason": "As r2 with the household income unit, to separate the two changes.", + "share": 0.16471154337166136, + "parent_plus_share": 0.21133541826981272, + "parent_plus_by_reference_age": { + "24_and_younger": 1.1813563767402388e-06, + "25_to_34": 0.002878811061981838, + "35_to_49": 0.03437594638384092, + "50_to_61": 0.4760763618611841, + "62_and_older": 0.7759316514071254 + }, + "by_head_age_band": { + "24_and_younger": 0.1992126713308297, + "25_to_34": 0.2819307615067069, + "35_to_49": 0.2452787877797859, + "50_to_61": 0.16522122503062336, + "62_and_older": 0.03883612510249803 + }, + "by_income_quintile": { + "1": 0.13542359408511098, + "2": 0.1539092744367252, + "3": 0.1590952051097958, + "4": 0.23485548375745907, + "5": 0.14025371248135685 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.6838024750342213, + "mean_among_borrowers_error_pct": 4.439528929525149, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.04597736952235504, + "head_age_band_share_mae_pp": 2.6656550710525306, + "race_share_mae_pp": 1.196987234789608, + "income_quintile_share_mae_pp": 2.5224786119025513, + "borrower_balance_band_tvd": 0.03490274180911923, + "borrower_age_band_tvd": 0.05061741585504796 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16471154337166136, + "mean_among_borrowers": 42874.79958838091, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 865353807411.8154, + "parent_plus_share_of_total": 0.21133541826981272 + }, + "household_balance_bands": { + "under_5k": 0.11607469728946729, + "5k_to_10k": 0.13438073884084578, + "10k_to_20k": 0.18390441158440993, + "20k_to_40k": 0.2175213345370531, + "40k_to_60k": 0.11322429790687383, + "60k_to_80k": 0.09839303402018727, + "80k_to_100k": 0.040696765534088644, + "100k_to_200k": 0.06459753241505105, + "200k_plus": 0.031207187872023125 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.1992126713308297, + "mean_among_borrowers": 18109.488073489367, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.2819307615067069, + "mean_among_borrowers": 36270.682996325166, + "median_among_borrowers": 21300.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.2452787877797859, + "mean_among_borrowers": 44326.47466297097, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16522122503062336, + "mean_among_borrowers": 46739.99930437942, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.03883612510249803, + "mean_among_borrowers": 62130.09345503386, + "median_among_borrowers": 30000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15248141549588676, + "mean_among_borrowers": 43244.56985324743, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.2524562948063122, + "mean_among_borrowers": 43362.722253136395, + "median_among_borrowers": 27000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.1439203802335117, + "mean_among_borrowers": 38963.62666957163, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.17859030346275792, + "mean_among_borrowers": 45351.73574282507, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.13542359408511098, + "mean_among_borrowers": 42547.97877457715, + "median_among_borrowers": 19322.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1539092744367252, + "mean_among_borrowers": 39890.227134542816, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.1590952051097958, + "mean_among_borrowers": 35227.932190327105, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.23485548375745907, + "mean_among_borrowers": 43504.8501632244, + "median_among_borrowers": 27500.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.14025371248135685, + "mean_among_borrowers": 54083.17619953545, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 20998958.074320182, + "mean_balance": 41209.36878625729, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.12299613974060244, + "5k_to_10k": 0.13840505815198073, + "10k_to_20k": 0.18337471576757322, + "20k_to_40k": 0.2190436105053335, + "40k_to_60k": 0.11735011741410456, + "60k_to_80k": 0.09189840869829337, + "80k_to_100k": 0.03614263089100404, + "100k_to_200k": 0.061323665208594696, + "200k_plus": 0.029465653622513483 + }, + "age_bands": { + "24_and_younger": 0.04513484789969594, + "25_to_34": 0.2928507022678982, + "35_to_49": 0.3809737565496338, + "50_to_61": 0.20683966973224618, + "62_and_older": 0.07420102355052594 + }, + "graduate_share_of_own_original_principal": 0.5129961891047207, + "ibr_new_borrower_share": 0.6152171940032605 + } + } + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py index 747ccfc99e..e09789e140 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -125,6 +125,9 @@ "SCF_EDUCATION_LOAN_WHOSE_COLUMNS", "SCF_EDUCATION_LOAN_YEAR_COLUMNS", "SCF_WHOSE_EDUCATION_HOLDER", + "RecipientDefinition", + "SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION", + "SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION", "ScfPublicFile", "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS", @@ -751,6 +754,66 @@ def load_scf_2025_education_debt_donor( # --------------------------------------------------------------------------- # # Recipients # # --------------------------------------------------------------------------- # +_REFERENCE_PERSON_RULES = frozenset({"householder_couple", "oldest_adult"}) +_PARTNERED_RULES = frozenset({"spouse_or_partner", "married_spouse"}) +_CHILDREN_RULES = frozenset({"of_reference_or_partner", "of_reference"}) +_INCOME_UNITS = frozenset({"household", "spm_unit", "reference_couple_and_children"}) + + +@dataclass(frozen=True) +class RecipientDefinition: + """How the CPS side builds the SCF predictors for one household. + + Each field names the CPS construction of one SCF concept: + + - ``reference_person``: ``householder_couple`` takes the CPS householder + (lowest line number) and, when a spouse or cohabiting partner is present, + the man of a mixed-sex couple or else the older partner, which is how + the SCF orders its economically dominant couple. ``oldest_adult`` is the + auto-loan rule (the only adult, the man of a two-adult married couple, + else the oldest adult). + - ``partnered``: ``spouse_or_partner`` is SCF ``married`` (married or living + with a partner, X8023 in 1-2): a spouse (``A_SPOUSE``) or cohabiting + partner (``PECOHAB``) is present. ``married_spouse`` is ``A_MARITL`` 1-2. + - ``children``: ``of_reference_or_partner`` is SCF ``kids`` (children, + step-children and children-in-law of the reference person or partner, + any age, anywhere in the household); ``of_reference`` counts only the + reference person's own children through ``PEPAR1``/``PEPAR2``. + - ``income_unit``: the people whose incomes are summed. ``household`` sums + every member; ``spm_unit`` the reference person's SPM unit; + ``reference_couple_and_children`` the reference person, spouse or + partner and their children. + """ + + reference_person: str = "householder_couple" + partnered: str = "spouse_or_partner" + children: str = "of_reference_or_partner" + income_unit: str = "household" + + def __post_init__(self) -> None: + for value, allowed, name in ( + (self.reference_person, _REFERENCE_PERSON_RULES, "reference_person"), + (self.partnered, _PARTNERED_RULES, "partnered"), + (self.children, _CHILDREN_RULES, "children"), + (self.income_unit, _INCOME_UNITS, "income_unit"), + ): + if value not in allowed: + raise ValueError( + f"RecipientDefinition.{name} must be one of {sorted(allowed)}." + ) + + +#: The definition the stage uses (see the module docstring for the evidence). +SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION = RecipientDefinition() +#: The definition of the first #1073 receipt, kept for its before/after table. +SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION = RecipientDefinition( + reference_person="oldest_adult", + partnered="married_spouse", + children="of_reference", + income_unit="household", +) + + @dataclass(frozen=True) class _Recipients: """Household-ordered predictors and each household's head and spouse rows.""" @@ -760,19 +823,24 @@ class _Recipients: spouse_position: np.ndarray # -1 where no spouse or partner is present -def _resolve_partner_positions( - person: pd.DataFrame, head_positions: np.ndarray -) -> np.ndarray: - """Positional row of each head's spouse (A_SPOUSE) or partner (PECOHAB).""" - +def _line_lookup(person: pd.DataFrame) -> dict[tuple[object, int], int]: household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() - lookup = { + return { (household_id, int(line_number)): position for position, (household_id, line_number) in enumerate( zip(household, line, strict=True) ) } + + +def _resolve_partner_positions( + person: pd.DataFrame, head_positions: np.ndarray +) -> np.ndarray: + """Positional row of each head's spouse (A_SPOUSE) or partner (PECOHAB).""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + lookup = _line_lookup(person) pointers = [pd.to_numeric(person["A_SPOUSE"], errors="coerce").fillna(0).to_numpy()] if _OPTIONAL_PARTNER_COLUMN in person.columns: pointers.append( @@ -793,7 +861,122 @@ def _resolve_partner_positions( return result -def _recipients(frame: Frame) -> _Recipients: +def _householder_couple_positions( + person: pd.DataFrame, order: np.ndarray +) -> np.ndarray: + """The SCF-ordered reference person of each household, aligned to ``order``.""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(9_999).to_numpy() + position = np.arange(len(person)) + ranked = np.lexsort((position, line, household)) + first = np.ones(len(person), dtype=bool) + first[1:] = household[ranked][1:] != household[ranked][:-1] + householders = ranked[first] + by_household = pd.Series(householders, index=household[householders]).reindex(order) + if by_household.isna().any(): + missing_ids = order[by_household.isna().to_numpy()][:5].tolist() + raise ValueError( + "SCF education-debt receiver household alignment failed: " + f"no householder for household id(s) {missing_ids}." + ) + householder = by_household.to_numpy(dtype=np.int64) + partner = _resolve_partner_positions(person, householder) + female = person["is_female"].astype(bool).to_numpy() + age = pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + has_partner = partner >= 0 + safe_partner = np.where(has_partner, partner, householder) + mixed = female[householder] != female[safe_partner] + partner_first = np.where( + mixed, + female[householder], # the man of a mixed-sex couple + age[safe_partner] > age[householder], # else the older partner + ) + return np.where(has_partner & partner_first, safe_partner, householder) + + +def _children_counts( + person: pd.DataFrame, + head_position: np.ndarray, + spouse_position: np.ndarray, + *, + include_partner: bool, +) -> np.ndarray: + """Household members whose CPS parent pointer names the head (or partner).""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + parent_of: dict[tuple[object, int], int] = { + (household[head], int(line[head])): index + for index, head in enumerate(head_position) + } + if include_partner: + for index, spouse in enumerate(spouse_position): + if spouse >= 0: + parent_of.setdefault((household[spouse], int(line[spouse])), index) + counts = np.zeros(len(head_position), dtype=np.float64) + pointers = [ + pd.to_numeric(person[column], errors="coerce").fillna(-1).to_numpy() + for column in ("PEPAR1", "PEPAR2") + ] + for row in range(len(person)): + owners = { + parent_of.get((household[row], int(pointer[row]))) + for pointer in pointers + if pointer[row] > 0 + } + owners.discard(None) + # A child of both partners counts once. + if owners: + counts[next(iter(owners))] += 1.0 + return counts + + +def _income_unit_owner( + person: pd.DataFrame, + head_position: np.ndarray, + spouse_position: np.ndarray, + unit: str, +) -> np.ndarray: + """For each person, the household row whose income unit includes them, or -1.""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + if unit == "household": + index = pd.Series(np.arange(len(head_position)), index=household[head_position]) + return index.reindex(household).to_numpy(dtype=np.int64) + owner = np.full(len(person), -1, dtype=np.int64) + if unit == "spm_unit": + if "person_spm_unit_id" not in person.columns: + raise ValueError("The spm_unit income unit requires person_spm_unit_id.") + spm = person["person_spm_unit_id"].to_numpy() + index = pd.Series(np.arange(len(head_position)), index=spm[head_position]) + index = index[~index.index.duplicated()] + mapped = index.reindex(spm).to_numpy(dtype=np.float64) + owner = np.where(np.isnan(mapped), -1, mapped).astype(np.int64) + owner[head_position] = np.arange(len(head_position)) + has_spouse = spouse_position >= 0 + owner[spouse_position[has_spouse]] = np.flatnonzero(has_spouse) + if unit == "reference_couple_and_children": + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + parent_of = { + (household[row], int(line[row])): owner[row] + for row in np.flatnonzero(owner >= 0) + } + for column in ("PEPAR1", "PEPAR2"): + pointer = ( + pd.to_numeric(person[column], errors="coerce").fillna(-1).to_numpy() + ) + for row in np.flatnonzero((owner < 0) & (pointer > 0)): + found = parent_of.get((household[row], int(pointer[row]))) + if found is not None: + owner[row] = found + return owner + + +def _recipients( + frame: Frame, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, +) -> _Recipients: person = frame.table("person") household = frame.table("household") missing = sorted(set(_REQUIRED_RECIPIENT_PERSON_COLUMNS) - set(person.columns)) @@ -805,66 +988,89 @@ def _recipients(frame: Frame) -> _Recipients: if "household_id" not in household.columns: raise ValueError("US SCF education-debt imputation requires household_id.") - mask = _scf_reference_person_mask(person) - head_rows = np.flatnonzero(mask) - household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() - head_by_household = pd.Series(head_rows, index=household_ids[head_rows]) - if head_by_household.index.duplicated().any(): - raise ValueError("SCF education-debt receiver selected two reference persons.") order = household["household_id"].to_numpy() - aligned = head_by_household.reindex(order) - if aligned.isna().any(): - missing_ids = order[aligned.isna().to_numpy()][:5].tolist() - raise ValueError( - "SCF education-debt receiver household alignment failed: " - f"no reference person for household id(s) {missing_ids}." - ) - head_position = aligned.to_numpy(dtype=np.int64) + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + if definition.reference_person == "householder_couple": + head_position = _householder_couple_positions(person, order) + else: + mask = _scf_reference_person_mask(person) + head_rows = np.flatnonzero(mask) + head_by_household = pd.Series(head_rows, index=household_ids[head_rows]) + if head_by_household.index.duplicated().any(): + raise ValueError( + "SCF education-debt receiver selected two reference persons." + ) + aligned = head_by_household.reindex(order) + if aligned.isna().any(): + missing_ids = order[aligned.isna().to_numpy()][:5].tolist() + raise ValueError( + "SCF education-debt receiver household alignment failed: " + f"no reference person for household id(s) {missing_ids}." + ) + head_position = aligned.to_numpy(dtype=np.int64) + spouse_position = _resolve_partner_positions(person, head_position) - per_person = pd.DataFrame(index=np.arange(len(person))) - per_person["age"] = ( + features = pd.DataFrame(index=np.arange(len(head_position))) + features["age"] = ( pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + )[head_position] + features["is_female"] = ( + person["is_female"].astype(bool).astype(float).to_numpy()[head_position] ) - per_person["is_female"] = person["is_female"].astype(bool).astype(float).to_numpy() - per_person["cps_race"] = _recipient_cps_race(person) - per_person["is_married"] = ( - pd.to_numeric(person["A_MARITL"], errors="coerce") - .isin([1, 2]) - .astype(float) - .to_numpy() - ) - per_person["own_children_in_household"] = _own_children_in_household(person) + features["cps_race"] = _recipient_cps_race(person)[head_position] + if definition.partnered == "spouse_or_partner": + features["is_married"] = (spouse_position >= 0).astype(np.float64) + else: + features["is_married"] = ( + pd.to_numeric(person["A_MARITL"], errors="coerce") + .isin([1, 2]) + .astype(float) + .to_numpy()[head_position] + ) + if definition.children == "of_reference_or_partner": + features["own_children_in_household"] = _children_counts( + person, head_position, spouse_position, include_partner=True + ) + else: + features["own_children_in_household"] = _own_children_in_household(person)[ + head_position + ] - income = pd.DataFrame( - { - "household_id": household_ids, - "employment_income": np.maximum( + owner = _income_unit_owner( + person, head_position, spouse_position, definition.income_unit + ) + member = owner >= 0 + for column, values in ( + ( + "employment_income", + np.maximum( pd.to_numeric(person["employment_income_before_lsr"], errors="coerce") .fillna(0.0) .to_numpy(dtype=np.float64), 0.0, ), - "interest_dividend_income": _sum_present( - person, _RECIPIENT_INTEREST_DIVIDEND_COLUMNS - ), - "social_security_pension_income": _sum_present( - person, _RECIPIENT_SS_PENSION_COLUMNS - ), - "household_income": _sum_present( - person, _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS - ), - } - ) - household_income = income.groupby("household_id", sort=False).sum() - features = per_person.iloc[head_position].reset_index(drop=True) - sums = household_income.reindex(order) - for column in household_income.columns: - features[column] = sums[column].to_numpy(dtype=np.float64) + ), + ( + "interest_dividend_income", + _sum_present(person, _RECIPIENT_INTEREST_DIVIDEND_COLUMNS), + ), + ( + "social_security_pension_income", + _sum_present(person, _RECIPIENT_SS_PENSION_COLUMNS), + ), + ( + "household_income", + _sum_present(person, _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS), + ), + ): + sums = np.zeros(len(head_position), dtype=np.float64) + np.add.at(sums, owner[member], values[member]) + features[column] = sums features.index = household.index return _Recipients( features=features.loc[:, list(SCF_EDUCATION_DEBT_PREDICTORS)], head_position=head_position, - spouse_position=_resolve_partner_positions(person, head_position), + spouse_position=spouse_position, ) @@ -1000,19 +1206,25 @@ def impute_us_scf_education_debt( *, seed: int, n_estimators: int = _DEFAULT_N_ESTIMATORS, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + predictors: Sequence[str] = SCF_EDUCATION_DEBT_PREDICTORS, ) -> pd.DataFrame: """Impute the four person columns onto a US frame. Returns a DataFrame indexed like the person table with :data:`US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS`; only reference persons and - their spouses or partners can be nonzero. + their spouses or partners can be nonzero. ``definition`` selects the CPS + construction of the SCF predictors and ``predictors`` the subset the QRFs + condition on; the receipts pass the first receipt's choices for their + before/after table. """ - recipients = _recipients(frame) + recipients = _recipients(frame, definition) drawn = draw_scf_education_debt_households( recipients.features, donor, seed=seed, + predictors=predictors, n_estimators=n_estimators, ) person = frame.table("person") @@ -1090,6 +1302,7 @@ def with_us_scf_education_debt_inputs( time_period: int, scf_education_debt_donor: pd.DataFrame, n_estimators: int = _DEFAULT_N_ESTIMATORS, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, ) -> Frame: """Write the four education-debt person inputs; idempotent on a signal.""" @@ -1105,6 +1318,7 @@ def with_us_scf_education_debt_inputs( scf_education_debt_donor, seed=int(seed), n_estimators=int(n_estimators), + definition=definition, ) tables = {entity: frame.table(entity).copy() for entity in frame.entities} for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS: @@ -1407,17 +1621,20 @@ def _racecl4_from_cps(person: pd.DataFrame) -> np.ndarray: return group -def us_scf_education_debt_fit_table(frame: Frame) -> dict[str, object]: +def us_scf_education_debt_fit_table( + frame: Frame, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, +) -> dict[str, object]: """The microcosm#1073 fit table of an imputed US frame, at its weights. - Households are grouped by the SCF-style reference person's age and + Households are grouped by the reference person's age and ``racecl4``-equivalent race, and by weighted quintiles of the household income the imputation conditions on (income quintiles of the frame's own - households, like the SCF's). + households, like the SCF's), all under ``definition``. """ person = frame.table("person") - recipients = _recipients(frame) + recipients = _recipients(frame, definition) household_weight = np.asarray( frame.weights_for("household").values, dtype=np.float64 ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py index 0e2e111bed..249c273984 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py @@ -669,3 +669,116 @@ def test_donor_fit_table_tabulates_observed_and_imputed_rows_alike() -> None: assert observed["households"]["share_with_federal_education_debt"] == ( pytest.approx(expected_share) ) + + +# --------------------------------------------------------------------------- # +# Recipient definitions # +# --------------------------------------------------------------------------- # +def _household(members: list[dict[str, object]], household_id: int = 1) -> list[dict]: + rows = [] + for index, member in enumerate(members, start=1): + rows.append( + { + "person_id": household_id * 100 + index, + "person_household_id": household_id, + "PH_SEQ": household_id, + "A_LINENO": index, + "A_SPOUSE": 0, + "PECOHAB": 0, + "age": 40.0, + "is_female": False, + "PRDTRACE": 1, + "PRDTHSP": 0, + "A_MARITL": 7, + "PEPAR1": -1, + "PEPAR2": -1, + "employment_income_before_lsr": 0.0, + **member, + } + ) + return rows + + +def _definition_frame() -> Frame: + rows = [ + # 1: a 30-year-old woman householder with her cohabiting partner (31), + # his child, and her 62-year-old mother. + *_household( + [ + {"age": 30.0, "is_female": True, "PECOHAB": 2, + "employment_income_before_lsr": 40_000.0}, + {"age": 31.0, "PECOHAB": 1, "employment_income_before_lsr": 50_000.0}, + {"age": 8.0, "PEPAR1": 2}, + {"age": 62.0, "is_female": True, + "employment_income_before_lsr": 30_000.0}, + ], + household_id=1, + ), + # 2: a married same-sex couple, the householder younger. + *_household( + [ + {"age": 35.0, "A_SPOUSE": 2, "A_MARITL": 1}, + {"age": 45.0, "A_SPOUSE": 1, "A_MARITL": 1}, + ], + household_id=2, + ), + ] # fmt: skip + person = pd.DataFrame.from_records(rows) + return _us_frame(person) + + +def test_recipient_definition_refuses_unknown_rules() -> None: + with pytest.raises(ValueError, match="reference_person"): + education_debt.RecipientDefinition(reference_person="eldest") + with pytest.raises(ValueError, match="income_unit"): + education_debt.RecipientDefinition(income_unit="tax_unit") + + +def test_default_definition_follows_the_scf_definitions() -> None: + frame = _definition_frame() + recipients = education_debt._recipients(frame) + person = frame.table("person") + features = recipients.features + + # The householder's cohabiting partner is the man of the couple; the + # same-sex couple's older spouse leads. + assert person["age"].to_numpy()[recipients.head_position].tolist() == [31.0, 45.0] + assert features["is_married"].tolist() == [1.0, 1.0] + # Household 1 counts the partner's child; the mother is no one's child. + assert features["own_children_in_household"].tolist() == [1.0, 0.0] + assert features["employment_income"].tolist() == [120_000.0, 0.0] + + +def test_first_receipt_definition_reproduces_the_oldest_adult_rule() -> None: + frame = _definition_frame() + recipients = education_debt._recipients( + frame, education_debt.SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION + ) + person = frame.table("person") + + assert person["age"].to_numpy()[recipients.head_position].tolist() == [62.0, 45.0] + assert recipients.features["is_married"].tolist() == [0.0, 1.0] + assert recipients.features["own_children_in_household"].tolist() == [0.0, 0.0] + + +def test_income_units_sum_the_declared_members() -> None: + frame = _definition_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + # The mother forms her own SPM unit. + spm = tables["person"]["person_spm_unit_id"].to_numpy().copy() + spm[3] = 9_999 + tables["person"]["person_spm_unit_id"] = spm + tables["spm_unit"] = pd.DataFrame({"spm_unit_id": np.unique(spm)}) + frame = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + ) + for unit, expected in ( + ("household", 120_000.0), + ("spm_unit", 90_000.0), + ("reference_couple_and_children", 90_000.0), + ): + definition = education_debt.RecipientDefinition(income_unit=unit) + features = education_debt._recipients(frame, definition).features + assert features["employment_income"].iloc[0] == expected, unit From b9f3d671912471bfd4070f127e08634c5fd3c4fe Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Tue, 20 Oct 2026 11:34:58 -0400 Subject: [PATCH 04/17] Condition the education-debt QRFs on SCF-comparable predictors (#1073) The first host fit drew federal education debt for 14.4% of CPS households against 18.2% in SCF 2025, and gave Parent PLUS 21.5% of balances against 13.8%, while the SCF hold-out reproduced the SCF. The cause is covariate shift: the CPS host carries tax-record imputations of interest and dividends and of Social Security and pensions, which put 42% and 18% of 25-34 households above zero against 16% and 6% of SCF families. Mapping each to its SCF distribution at equal rank moves the host's borrower share most of any predictor; all together reach the hold-out level (receipts/1073-predictor-shift-diagnosis.json). - SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS keeps every predictor both sides build; SCF_EDUCATION_DEBT_PREDICTORS, used by both QRFs, leaves out the two shifted income components, each with its reason in SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS. - The recipient definition sums income over the reference person, spouse or partner and their children, the CPS counterpart of the SCF primary economic unit, on top of the SCF-matching codings. - The stage spec drift check now covers both QRFs; the stage notes, module docstring and changelog say why. - The receipt builder reports the build, the first receipt and the all-candidates setting side by side and records commit and run id. - experiments/modal_1073.py runs the heavy steps on Modal, adds generate and spec steps, and collects every tracked file a run changed. Re-pins the frozen source_stages.json digest; the generated mirrors are regenerated on Modal in a follow-up commit. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-2025-education-debt.added.md | 4 +- .../build_1073_scf_education_debt_receipt.py | 176 ++++++- experiments/diagnose_1073_predictor_shift.py | 15 +- .../explore_1073_recipient_definitions.py | 4 +- experiments/modal_1073.py | 464 ++++++++++++++++++ .../1073-recipient-definition-variants.json | 98 ++-- .../src/microcosm/build/us/source_stages.json | 6 +- .../microcosm/build/us_runtime/__init__.py | 6 + .../build/us_runtime/scf_education_debt.py | 121 +++-- .../us/test_us_bundle_core_contracts.py | 2 +- .../us/test_us_scf_education_debt.py | 43 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/generate_us_bundle_from_constants.py | 2 +- 13 files changed, 826 insertions(+), 117 deletions(-) create mode 100644 experiments/modal_1073.py diff --git a/changelog.d/1073-scf-2025-education-debt.added.md b/changelog.d/1073-scf-2025-education-debt.added.md index 493506b44d..3b845e931f 100644 --- a/changelog.d/1073-scf-2025-education-debt.added.md +++ b/changelog.d/1073-scf-2025-education-debt.added.md @@ -4,8 +4,8 @@ Only education debt reads the 2025 SCF. Net worth, the liquid-asset leaves and a The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`; loans for a child go to the reference person's `federal_parent_plus_loan_balance`; codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. A person is an IBR new borrower when every federal loan they hold was borrowed in 2014 or later, and the graduate share weights each own loan's program by the amount borrowed. -The model. One weighted QRF draws each household's federal education debt on the eight shared SCF predictors plus household income less capital gains. A second, fit on borrower families and conditioning on that total, draws the holder shares, graduate shares and new-borrower flags. The SCF-style reference person and the member its `A_SPOUSE` (else `PECOHAB`) pointer names receive the balances. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. +The model. One weighted QRF draws each household's federal education debt. A second, fit on borrower families and conditioning on that total, draws the holder shares, graduate shares and new-borrower flags. Both condition on the reference person's age, sex and race, a spouse or partner being present, the children of the reference person or partner, and the wages and income less capital gains of the reference person, spouse or partner and their children. The CPS side codes each predictor the way the SCF defines it: SCF `MARRIED` counts cohabiting partners, SCF `KIDS` counts the partner's children, the reference person comes from the householder couple (the man, or the older partner), and income covers the primary economic unit rather than the whole household. The shared interest-dividend and Social Security-pension predictors are left out (`SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS`). The CPS host carries tax-record imputations of both, so at reference ages 25-34 it reports them for 42% and 18% of households against 16% and 6% of SCF families, and that shift was the main reason the first host fit drew too few borrowers. The reference person and the member its `A_SPOUSE` (else `PECOHAB`) pointer names receive the balances. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. The fiscal-refresh release tool runs the stage only with `--scf-education-debt`, because the pinned policyengine-us does not define the four inputs yet. The four columns join `US_STORED_NON_VARIABLE_COLUMNS` with the pending-engine reason the parent ids carry, and leave it when the engine pin moves. They are not in the release input-coverage manifest, whose check requires every column to be an engine input. No release is certified with them. -Fit receipts are in `experiments/1073-scf-education-debt-receipts.md`, generated by `experiments/build_1073_scf_education_debt_receipt.py`. +Fit receipts are in `experiments/1073-scf-education-debt-receipts.md`, generated by `experiments/build_1073_scf_education_debt_receipt.py` on Modal through `experiments/modal_1073.py`, which clones a pushed commit, verifies every input's SHA-256 and records the run id in the receipt. The diagnosis behind the predictor and recipient choices is in `experiments/receipts/1073-predictor-shift-diagnosis.json` and `1073-recipient-definition-variants.json`. diff --git a/experiments/build_1073_scf_education_debt_receipt.py b/experiments/build_1073_scf_education_debt_receipt.py index 7c8068b005..186eff7186 100644 --- a/experiments/build_1073_scf_education_debt_receipt.py +++ b/experiments/build_1073_scf_education_debt_receipt.py @@ -11,11 +11,17 @@ by ``yy1`` (all implicates of a family stay together). Each fold is drawn from the stage's two QRFs fit on the other four, and the drawn table is compared with the observed one, for each predictor set. -3. **Host fit.** The stage imputed onto a US Microcosm H5 (the published - default unless ``--h5`` names another), tabulated at that file's weights. - -Nothing here is a gate or a calibration target. Run with the repository's -engine-free environment: +3. **Host fit, before and after.** The stage imputed onto a US Microcosm H5 + (the published default unless ``--h5`` names another), tabulated at that + file's weights, under three settings (``HOST_SETTINGS``): the first receipt's + construction, the SCF-matching recipient definition with every candidate + predictor, and the build (that definition without the two shifted income + predictors). + +Nothing here is a gate or a calibration target. The committed receipts are +produced on Modal by ``experiments/modal_1073.py`` (step ``receipt``), which +clones a pushed commit and verifies every input's SHA-256; the receipt records +that commit and the run id. A local run needs about 4 GB: .venv/Scripts/python.exe experiments/build_1073_scf_education_debt_receipt.py \ --scf-zip-dir ~/.cache/scf --h5 populace_us_2024.h5 @@ -28,6 +34,7 @@ import json import zipfile from collections.abc import Mapping, Sequence +from dataclasses import asdict from datetime import UTC, datetime from pathlib import Path @@ -39,16 +46,21 @@ SCF_2025_SUMMARY_EXTRACT, SCF_EDUCATION_DEBT_BALANCE_BANDS, SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, SCF_EDUCATION_DEBT_TOTAL_TARGET, SCF_EDUCATION_LOAN_BALANCE_COLUMNS, SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + RecipientDefinition, ScfPublicFile, _read_stata_columns, draw_scf_education_debt_households, fetch_scf_public_file, + impute_us_scf_education_debt, load_scf_2025_education_debt_donor, scf_education_debt_donor_fit_table, us_scf_education_debt_fit_table, @@ -71,12 +83,38 @@ #: SHA-256 (releases/populace-us-2024-spm-20260915/release_manifest.json). DEFAULT_HOST_RELEASE = "populace-us-2024-spm-20260915" DEFAULT_HOST_SHA256 = "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" -PRODUCTION_VARIANT = "shared_8_plus_household_income" +#: The build's predictor set, and the first receipt's (every candidate). +PRODUCTION_VARIANT = "build_without_interest_and_ss_pension" +FIRST_RECEIPT_VARIANT = "shared_8_plus_household_income" VARIANTS: Mapping[str, tuple[str, ...]] = { "shared_8_without_age": tuple(p for p in SCF_WEALTH_PREDICTORS if p != "age"), "shared_8": SCF_WEALTH_PREDICTORS, + FIRST_RECEIPT_VARIANT: SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + "plus_household_income_and_education": ( + *SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + "educ", + ), PRODUCTION_VARIANT: SCF_EDUCATION_DEBT_PREDICTORS, - "plus_household_income_and_education": (*SCF_EDUCATION_DEBT_PREDICTORS, "educ"), + "build_plus_education": (*SCF_EDUCATION_DEBT_PREDICTORS, "educ"), +} +#: Host settings for the before/after table: (label, definition, predictors). +BUILD_SETTING = "build" +HOST_SETTINGS: Mapping[str, tuple[str, RecipientDefinition, tuple[str, ...]]] = { + "first_receipt": ( + "First receipt", + SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + ), + "scf_codings_all_candidates": ( + "SCF codings, all predictors", + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + ), + BUILD_SETTING: ( + "Build", + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_PREDICTORS, + ), } WHOSE_LABELS = { 1: "1 respondent (reference person)", @@ -107,6 +145,19 @@ ) +def _git_commit() -> str: + import subprocess + + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=Path(__file__).resolve().parent, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -369,9 +420,10 @@ def markdown(receipt: Mapping[str, object]) -> str: observed = receipt["scf_2025_observed"] holdout = receipt["holdout"]["variants"][PRODUCTION_VARIANT]["fit"] host = receipt.get("host", {}).get("fit") - columns = [("SCF 2025", observed), ("SCF hold-out draw", holdout)] - if host is not None: - columns.append((f"Imputed onto {receipt['host']['release']}", host)) + settings = receipt.get("host_settings", {}) + columns = [("SCF 2025", observed), ("SCF hold-out (build)", holdout)] + for setting in settings.values(): + columns.append((f"Host: {setting['label']}", setting["fit"])) header = "| | " + " | ".join(label for label, _ in columns) + " |" rule = "|---|" + "---:|" * len(columns) @@ -387,13 +439,30 @@ def row(label: str, getter, formatter) -> str: "", f"Generated {receipt['generated_at']} by " "`experiments/build_1073_scf_education_debt_receipt.py` on branch " - "`scf-2025-education-debt`. Machine-readable copy: " + f"`scf-2025-education-debt`, commit `{receipt['provenance']['commit']}`" + + ( + f", Modal run `{receipt['provenance']['run']}` " + "(`experiments/modal_1073.py`)" + if receipt["provenance"].get("run") + else "" + ) + + ". Machine-readable copy: " "`experiments/receipts/1073-scf-education-debt-fit.json`.", "", + "Host columns impute the published default " + f"`{receipt.get('host', {}).get('release', DEFAULT_HOST_RELEASE)}` at its " + "calibrated weights under three settings: the first receipt's construction " + "(oldest adult, `A_MARITL` marriage, own children, whole-household income, " + "every candidate predictor); the SCF-matching codings with every candidate " + "predictor; and the build, which also leaves out interest and dividends and " + "Social Security and pensions. The cause and the evidence are in the " + "stage notes and `experiments/receipts/1073-predictor-shift-diagnosis.json`.", + "", "Every figure is weighted. SCF rows pool the five implicates with `wgt` " "(X42001/5). Household figures group by the reference person's age and " - "race and by weighted quintiles of household income less capital gains, " - "the income concept the imputation conditions on. Borrower figures count " + "race and by weighted quintiles of income less capital gains of the " + "setting's income unit, the income concept that setting conditions on. " + "Borrower figures count " "people with a federal balance (own plus Parent PLUS), in Federal Student " "Aid's balance and age bands; SCF borrowers are the reference person and " "spouse. Nothing here is a gate or a calibration target.", @@ -560,22 +629,42 @@ def row(label: str, getter, formatter) -> str: + " |" ) if host is not None: + host_metrics = [m for m in metrics if m != "household_rank_correlation"] lines += [ "", - "## Host fit", + "## Host fit, before and after", "", f"Imputed with build seed {receipt['host']['seed']} onto " f"`{receipt['host']['release']}` ({receipt['host']['households']:,} " f"households, sha256 `{receipt['host']['sha256'][:12]}...`) at its " "calibrated household weights; the published weights were calibrated " - "without these columns. Signal gate: " - f"{'passed' if receipt['host']['gate_passed'] else 'FAILED'}.", + "without these columns. Build signal gate: " + f"{'passed' if receipt['host']['gate_passed'] else 'FAILED'}. Errors " + "are imputed minus SCF 2025.", "", - "| Metric (imputed minus SCF 2025) | Value |", - "|---|---:|", + "| Setting | Households with debt | Parent PLUS share | " + + " | ".join(host_metrics) + + " |", + "|---|---:|---:|" + "---:|" * len(host_metrics), ] - for key, value in receipt["host"]["metrics_vs_scf"].items(): - lines.append(f"| {key} | {value:.3f} |") + for setting in settings.values(): + fit = setting["fit"] + lines.append( + f"| {setting['label']} | " + f"{_pct(fit['households']['share_with_federal_education_debt'])} | " + f"{_pct(fit['households']['parent_plus_share_of_total'])} | " + + " | ".join( + f"{setting['metrics_vs_scf'][m]:.3f}" for m in host_metrics + ) + + " |" + ) + lines.append( + "| SCF 2025 | " + f"{_pct(observed['households']['share_with_federal_education_debt'])} | " + f"{_pct(observed['households']['parent_plus_share_of_total'])} | " + + " | ".join("" for _ in host_metrics) + + " |" + ) lines.append("") return "\n".join(lines) @@ -594,6 +683,11 @@ def main(argv: Sequence[str] | None = None) -> int: ) parser.add_argument("--h5", type=Path, help="US Microcosm H5 to impute onto.") parser.add_argument("--host-release", default=DEFAULT_HOST_RELEASE) + parser.add_argument( + "--run", + default="", + help="Label of the run that produced the receipt (the Modal run id).", + ) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--folds", type=int, default=5) parser.add_argument("--n-estimators", type=int, default=100) @@ -648,6 +742,7 @@ def main(argv: Sequence[str] | None = None) -> int: "schema_version": 1, "issue": ISSUE, "generated_at": datetime.now(UTC).strftime("%Y-%m-%d"), + "provenance": {"commit": _git_commit(), "run": args.run}, "scf_files": { public_file.member: { "url": public_file.url, @@ -691,9 +786,38 @@ def main(argv: Sequence[str] | None = None) -> int: if args.h5 is not None: host_sha256 = _sha256(args.h5) - frame = load_host_frame(args.h5) + base = load_host_frame(args.h5) + receipt["host_settings"] = {} + for name, (label, definition, predictors) in HOST_SETTINGS.items(): + if name == BUILD_SETTING: + continue + imputed = impute_us_scf_education_debt( + base, + donor, + seed=args.seed, + n_estimators=args.n_estimators, + definition=definition, + predictors=predictors, + ) + tables = {entity: base.table(entity).copy() for entity in base.entities} + for column in imputed.columns: + tables["person"][column] = imputed[column].to_numpy() + setting_frame = Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + ) + setting_fit = us_scf_education_debt_fit_table(setting_frame, definition) + receipt["host_settings"][name] = { + "label": label, + "definition": asdict(definition), + "predictors": list(predictors), + "fit": setting_fit, + "metrics_vs_scf": fit_metrics(observed, setting_fit), + } + # The build runs through the stage entry point and its signal gate. frame = with_us_scf_education_debt_inputs( - frame, + base, seed=args.seed, time_period=2024, scf_education_debt_donor=donor, @@ -701,6 +825,14 @@ def main(argv: Sequence[str] | None = None) -> int: ) gate = us_scf_education_debt_signal_gate(frame) host_fit = us_scf_education_debt_fit_table(frame) + label, definition, predictors = HOST_SETTINGS[BUILD_SETTING] + receipt["host_settings"][BUILD_SETTING] = { + "label": label, + "definition": asdict(definition), + "predictors": list(predictors), + "fit": host_fit, + "metrics_vs_scf": fit_metrics(observed, host_fit), + } receipt["host"] = { "release": args.host_release, "path": args.h5.name, diff --git a/experiments/diagnose_1073_predictor_shift.py b/experiments/diagnose_1073_predictor_shift.py index 44384e2f2e..66933267b8 100644 --- a/experiments/diagnose_1073_predictor_shift.py +++ b/experiments/diagnose_1073_predictor_shift.py @@ -23,9 +23,9 @@ import pandas as pd from microcosm.build.us_runtime.scf_education_debt import ( - SCF_EDUCATION_DEBT_PREDICTORS, - SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, SCF_EDUCATION_DEBT_TOTAL_TARGET, + RecipientDefinition, _recipients, _stage_seed, load_scf_2025_education_debt_donor, @@ -38,6 +38,11 @@ receipt = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(receipt) +#: The construction this diagnosis measured: the SCF-matching codings with +#: whole-household income, before the predictor and income-unit remedies it led +#: to (receipt run 20261020T150445Z-47949111f). Pinned so a rerun keeps its +#: meaning now that the stage default has moved on. +DIAGNOSED_DEFINITION = RecipientDefinition(income_unit="household") BANDS = (("18_24", 18, 24), ("25_34", 25, 34), ("35_49", 35, 49), ("50_61", 50, 61), ("62_plus", 62, 200)) # fmt: skip CONTINUOUS = ( @@ -124,7 +129,7 @@ def main(argv: Sequence[str] | None = None) -> int: args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" ) frame = receipt.load_host_frame(args.h5) - host = _recipients(frame, SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION).features + host = _recipients(frame, DIAGNOSED_DEFINITION).features host_w = np.asarray(frame.weights_for("household").values, dtype=float) host_age = host["age"].to_numpy(float) scf_w = donor["scf_weight"].to_numpy(float) @@ -135,7 +140,7 @@ def main(argv: Sequence[str] | None = None) -> int: h = (host_age >= lo) & (host_age <= hi) s = (scf_age >= lo) & (scf_age <= hi) entry = {} - for column in SCF_EDUCATION_DEBT_PREDICTORS: + for column in SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS: if column == "age": continue binary = column in ("is_female", "is_married") @@ -157,7 +162,7 @@ def main(argv: Sequence[str] | None = None) -> int: } # fmt: skip report["distributions"][band] = entry - predictors = list(SCF_EDUCATION_DEBT_PREDICTORS) + predictors = list(SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS) fit = donor.loc[:, [*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET, "scf_weight"]] model = QRF(n_estimators=100, seed=_stage_seed(0, 1)).fit( fit, diff --git a/experiments/explore_1073_recipient_definitions.py b/experiments/explore_1073_recipient_definitions.py index 8840a228cf..9107129ef5 100644 --- a/experiments/explore_1073_recipient_definitions.py +++ b/experiments/explore_1073_recipient_definitions.py @@ -24,8 +24,8 @@ from microcosm.build.us_runtime.scf_education_debt import ( SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, - SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_TOTAL_TARGET, RecipientDefinition, _recipients, @@ -81,7 +81,7 @@ "The PEU narrowed to the couple and their children.", ), ) -_ALL = tuple(SCF_EDUCATION_DEBT_PREDICTORS) +_ALL = tuple(SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS) _NO_INTDIV = tuple(p for p in _ALL if p != "interest_dividend_income") _NO_INTDIV_SSRET = tuple(p for p in _NO_INTDIV if p != "social_security_pension_income") #: Predictor-set remedies on top of the couple income unit. Each row also gets diff --git a/experiments/modal_1073.py b/experiments/modal_1073.py new file mode 100644 index 0000000000..c64ccd1cb2 --- /dev/null +++ b/experiments/modal_1073.py @@ -0,0 +1,464 @@ +"""Run the microcosm#1073 education-debt experiments on Modal instead of local RAM. + +Each run clones a pushed commit of PolicyEngine/microcosm, syncs that tree's +uv.lock, fetches the inputs and verifies each sha256, runs the requested steps +in order, and saves every receipt the steps write plus a run log to the +`microcosm-1073-cache` volume under results//. `collect` copies a +run's files back into this checkout. Inputs: + +- the published host release `populace-us-2024-spm-20260915` from Hugging Face + (policyengine/populace-us at a pinned revision); +- the Federal Reserve's SCF 2025 public zips (summary extract and full file). + + export PYTHONIOENCODING=utf-8 PYTHONUTF8=1 + modal run experiments/modal_1073.py::preflight --commit + modal run --detach experiments/modal_1073.py::main --commit --steps tests,shift,explore + modal run experiments/modal_1073.py::collect --run-id + +Steps (see STEPS; each runs one or more commands, stopping at the first that +fails): + +- ``generate``: regenerate the US spec bundle mirrors (``spec/*.yaml``, + ``country_package.json``, ``engine_abi.lock.json``) with + ``tools/generate_us_bundle_from_constants.py`` after a hand edit of + ``source_stages.json``; +- ``tests``: the stage's unit tests; +- ``spec``: the engine-free and engine-contract tests that read the spec + bundle, the stored-input register and the SCF stages, plus the release + tool's education-debt tests; +- ``hostfit``, ``shift``, ``explore``: the #1073 diagnostics; +- ``receipt``: the fit receipt, labelled with the run id. + +After the steps the run saves every tracked file they changed +(``git status --porcelain``) and every untracked file matching OUTPUT_GLOBS, +so ``collect`` brings regenerated mirrors back as well as receipts. This is an +experiment runner, not the registered stage runner in tools/modal_us_stage.py; +it writes nothing to the Hub and publishes nothing. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import time +import urllib.request +import zipfile +from datetime import UTC, datetime +from pathlib import Path + +import modal + +REPO_URL = "https://github.com/PolicyEngine/microcosm.git" +HOST = { + "repo_id": "policyengine/populace-us", + "filename": "populace_us_2024.h5", + "revision": "8ab57ffc2ca41d8af631ff62ebbce95e966299f3", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", +} +# zip name -> (url, zip sha256, member, member sha256) +SCF_ZIPS = { + "scfp2025s.zip": ( + "https://www.federalreserve.gov/econres/files/scfp2025s.zip", + "32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + "rscfp2025.dta", + "454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd", + ), + "scf2025s.zip": ( + "https://www.federalreserve.gov/econres/files/scf2025s.zip", + "c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + "p25i6.dta", + "38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad", + ), +} +_TESTS = "packages/microcosm-build/tests" +#: step -> commands. A part "glob:" expands to the sorted matches. +STEPS: dict[str, list[list[str]]] = { + "generate": [ + [ + "{python}", + "tools/generate_us_bundle_from_constants.py", + "--engine-lock-only", + ], + ["{python}", "tools/generate_us_bundle_from_constants.py"], + ], + "tests": [ + [ + "{python}", + "-m", + "pytest", + "-q", + f"{_TESTS}/engine_free/us/test_us_scf_education_debt.py", + ] + ], + "spec": [ + [ + "{python}", + "-m", + "pytest", + "-q", + f"glob:{_TESTS}/engine_free/shared/test_spec_engine_*.py", + f"{_TESTS}/engine_free/us/test_us_bundle_core_contracts.py", + f"{_TESTS}/engine_free/us/test_us_spec_bundle.py", + f"{_TESTS}/engine_free/us/test_us_stored_input_register.py", + f"{_TESTS}/engine_free/us/test_us_scf_auto_loans.py", + f"{_TESTS}/engine_free/us/test_us_scf_wealth.py", + f"{_TESTS}/engine_free/us/test_us_plan.py", + f"{_TESTS}/engine_free/us/test_us_spine_blindness.py", + ], + [ + "{python}", + "-m", + "pytest", + "-q", + f"{_TESTS}/engine_free/us/test_us_fiscal_refresh_builder.py", + "-k", + "scf_education_debt", + ], + [ + "{python}", + "-m", + "pytest", + "-q", + f"glob:{_TESTS}/engine_contract/us/test_spec_engine_*.py", + f"{_TESTS}/engine_contract/us/test_us_bundle_core_contracts.py", + f"{_TESTS}/engine_contract/us/test_us_spec_bundle.py", + f"{_TESTS}/engine_contract/us/test_us_stored_input_register.py", + f"{_TESTS}/engine_contract/us/test_us_scf_wealth.py", + f"{_TESTS}/engine_contract/us/test_imputation_lineage_spec.py", + f"{_TESTS}/engine_contract/us/test_release_input_coverage.py", + ], + ], + "hostfit": [ + [ + "{python}", + "experiments/diagnose_1073_host_fit.py", + "--h5", + "{h5}", + "--scf-cache-dir", + "{scf}", + ] + ], + "shift": [ + [ + "{python}", + "experiments/diagnose_1073_predictor_shift.py", + "--h5", + "{h5}", + "--scf-cache-dir", + "{scf}", + ] + ], + "explore": [ + [ + "{python}", + "experiments/explore_1073_recipient_definitions.py", + "--h5", + "{h5}", + "--scf-cache-dir", + "{scf}", + ] + ], + "receipt": [ + [ + "{python}", + "experiments/build_1073_scf_education_debt_receipt.py", + "--h5", + "{h5}", + "--scf-zip-dir", + "{zips}", + "--scf-cache-dir", + "{scf}", + "--run", + "{run_id}", + ] + ], +} +OUTPUT_GLOBS = ("experiments/receipts/1073-*.json", "experiments/1073-*.md") +CPU = 8.0 +MEMORY_MB = 32768 + +image = ( + modal.Image.debian_slim(python_version="3.14") + .apt_install("git") + .pip_install("uv==0.11.7", "huggingface_hub") +) +app = modal.App("microcosm-1073-experiments") +cache = modal.Volume.from_name("microcosm-1073-cache", create_if_missing=True) +CACHE = Path("/cache") +WORK = Path("/work/microcosm") +VENV = Path("/opt/venv") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def _verified(path: Path, sha256: str) -> bool: + return path.exists() and _sha256(path) == sha256 + + +def _checkout(commit: str) -> None: + WORK.mkdir(parents=True, exist_ok=True) + run = lambda *args: subprocess.run(args, cwd=WORK, check=True) # noqa: E731 + run("git", "init", "-q") + run("git", "remote", "add", "origin", REPO_URL) + run("git", "fetch", "-q", "--depth", "1", "origin", commit) + run("git", "checkout", "-q", "--detach", "FETCH_HEAD") + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=WORK, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if head != commit: + raise RuntimeError(f"checked out {head}, expected {commit}") + env = os.environ | { + "UV_PROJECT_ENVIRONMENT": str(VENV), + # A Modal volume cannot host uv's cache: uv persists temporary files + # by atomic rename, which the volume refuses. + "UV_CACHE_DIR": "/root/.cache/uv", + "UV_LINK_MODE": "copy", + } + subprocess.run( + ["uv", "sync", "--all-packages", "--extra", "us", "--frozen"], + cwd=WORK, + env=env, + check=True, + ) + + +def _inputs() -> dict[str, Path]: + from huggingface_hub import hf_hub_download + + host_dir = CACHE / "host" / HOST["revision"] + host = host_dir / HOST["filename"] + if not _verified(host, HOST["sha256"]): + hf_hub_download( + repo_id=HOST["repo_id"], + filename=HOST["filename"], + revision=HOST["revision"], + repo_type="dataset", + local_dir=host_dir, + ) + if not _verified(host, HOST["sha256"]): + raise RuntimeError("host H5 sha256 mismatch") + zips = CACHE / "scf" / "zips" + scf = CACHE / "scf" / "dta" + zips.mkdir(parents=True, exist_ok=True) + scf.mkdir(parents=True, exist_ok=True) + for name, (url, zip_sha, member, member_sha) in SCF_ZIPS.items(): + archive = zips / name + if not _verified(archive, zip_sha): + request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(request, timeout=600) as response: + archive.write_bytes(response.read()) + if not _verified(archive, zip_sha): + raise RuntimeError(f"{name} sha256 mismatch") + target = scf / member + if not _verified(target, member_sha): + with zipfile.ZipFile(archive) as bundle: + target.write_bytes(bundle.read(member)) + if not _verified(target, member_sha): + raise RuntimeError(f"{member} sha256 mismatch") + cache.commit() + return {"h5": host, "zips": zips, "scf": scf} + + +def _expand(command: list[str], fields: dict[str, str]) -> list[str]: + parts: list[str] = [] + for part in command: + if part.startswith("glob:"): + matches = sorted(WORK.glob(part.removeprefix("glob:"))) + if not matches: + raise RuntimeError(f"{part} matched nothing") + parts.extend(str(path.relative_to(WORK)) for path in matches) + else: + parts.append(part.format(**fields)) + return parts + + +def _save_changed_files(target_root: Path) -> list[str]: + """Copy tracked files the steps changed, and new outputs, under target_root.""" + + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=WORK, + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + saved = [] + for line in status: + code, rel = line[:2], line[3:] + path = WORK / rel + untracked = code == "??" + if untracked and not any(path.match(p) for p in OUTPUT_GLOBS): + continue + if not path.is_file(): + continue + target = target_root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(path.read_bytes()) + saved.append(f"{code} {rel}") + return saved + + +@app.function( + image=image, cpu=2.0, memory=8192, timeout=3600, volumes={"/cache": cache} +) +def preflight_remote(commit: str) -> dict: + """Clone, sync and stage the inputs without running a step.""" + started = time.time() + _checkout(commit) + paths = _inputs() + probe = subprocess.run( + [ + str(VENV / "bin" / "python"), + "-c", + "import microcosm.build.us_runtime.scf_education_debt as m; print(m.__file__)", + ], + cwd=WORK, + capture_output=True, + text=True, + ) + return { + "commit": commit, + "inputs": {key: str(path) for key, path in paths.items()}, + "import_ok": probe.returncode == 0, + "import_output": (probe.stdout + probe.stderr)[-2000:], + "seconds": round(time.time() - started, 1), + } + + +@app.function( + image=image, cpu=CPU, memory=MEMORY_MB, timeout=4 * 3600, volumes={"/cache": cache} +) +def run_steps(commit: str, steps: list[str], run_id: str) -> dict: + import resource # Unix-only; the local entrypoint runs on Windows too + + started = datetime.now(UTC) + _checkout(commit) + paths = _inputs() + fields = {key: str(path) for key, path in paths.items()} | { + "python": str(VENV / "bin" / "python"), + "run_id": run_id, + } + threads = str(int(CPU)) + env = os.environ | { + "PYTHONUNBUFFERED": "1", + "OMP_NUM_THREADS": threads, + "MKL_NUM_THREADS": threads, + "OPENBLAS_NUM_THREADS": threads, + } + out = CACHE / "results" / run_id + out.mkdir(parents=True, exist_ok=True) + log = { + "run_id": run_id, + "commit": commit, + "steps": [], + "cpu": CPU, + "memory_mb": MEMORY_MB, + "inputs": {"host": HOST, "scf": {k: v[1] for k, v in SCF_ZIPS.items()}}, + "started_at": started.isoformat(), + } + failed = False + for step in steps: + commands = [_expand(command, fields) for command in STEPS[step]] + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + t0 = time.time() + returncodes = [] + with (out / f"{step}.log").open("w", encoding="utf-8") as handle: + for command in commands: + handle.write("$ " + " ".join(command) + "\n") + handle.flush() + result = subprocess.run( + command, + cwd=WORK, + env=env, + stdout=handle, + stderr=subprocess.STDOUT, + text=True, + ) + returncodes.append(result.returncode) + if result.returncode != 0: + break + peak_kb = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + log["steps"].append( + { + "step": step, + "commands": commands, + "returncodes": returncodes, + "returncode": next((code for code in returncodes if code), 0), + "seconds": round(time.time() - t0, 1), + "peak_child_rss_gb": round(max(peak_kb, before) / 1024 / 1024, 2), + } + ) + cache.commit() + if any(returncodes): + failed = True + break + log["failed"] = failed + log["changed_files"] = _save_changed_files(out / "files") + log["finished_at"] = datetime.now(UTC).isoformat() + (out / "run.json").write_text(json.dumps(log, indent=2) + "\n", encoding="utf-8") + cache.commit() + return log + + +@app.function(image=image, volumes={"/cache": cache}) +def fetch_results(run_id: str) -> dict[str, bytes]: + root = CACHE / "results" / run_id + return { + str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file() + } + + +def _check_commit(commit: str) -> None: + if len(commit) != 40 or any(c not in "0123456789abcdef" for c in commit): + raise SystemExit("--commit must be a full 40-hex sha that is pushed to GitHub") + + +@app.local_entrypoint() +def preflight(commit: str) -> None: + _check_commit(commit) + print(json.dumps(preflight_remote.remote(commit), indent=2)) + + +@app.local_entrypoint() +def main(commit: str, steps: str = "tests,shift,explore") -> None: + _check_commit(commit) + names = [s for s in steps.split(",") if s] + unknown = sorted(set(names) - set(STEPS)) + if unknown: + raise SystemExit(f"unknown steps: {unknown}") + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + commit[:9] + print(f"run_id={run_id}") + print(json.dumps(run_steps.remote(commit, names, run_id), indent=2)) + + +@app.local_entrypoint() +def collect(run_id: str, logs_dir: str = "") -> None: + """Write a run's changed files into this checkout and its logs to logs_dir.""" + repo = Path(__file__).resolve().parents[1] + logs = ( + Path(logs_dir) + if logs_dir + else Path.home() / ".cache" / "microcosm" / "modal-1073" / run_id + ) + logs.mkdir(parents=True, exist_ok=True) + for rel, data in fetch_results.remote(run_id).items(): + if rel.startswith("files/"): + target = repo / rel.removeprefix("files/") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + print("file", target.relative_to(repo)) + else: + (logs / rel).write_bytes(data) + print("log", logs / rel) diff --git a/experiments/receipts/1073-recipient-definition-variants.json b/experiments/receipts/1073-recipient-definition-variants.json index f7211fcf68..a4cf4d0187 100644 --- a/experiments/receipts/1073-recipient-definition-variants.json +++ b/experiments/receipts/1073-recipient-definition-variants.json @@ -1,7 +1,7 @@ { "scf_2025": { "share": 0.18154956812200357, - "parent_plus_share": 0.13750075307389342, + "parent_plus_share": 0.13750075307389345, "parent_plus_by_reference_age": { "24_and_younger": 0.0, "25_to_34": 0.0036635524662026776, @@ -56,7 +56,7 @@ "household_income" ], "share": 0.17281627661600854, - "parent_plus_share": 0.11558498718060217, + "parent_plus_share": 0.11558498718060221, "metrics_vs_scf": { "share_with_debt_error_pp": -0.8733291505995028, "mean_among_borrowers_error_pct": -4.349507836301458, @@ -71,13 +71,13 @@ }, "reason": "The first receipt's construction.", "share": 0.14412999386354491, - "parent_plus_share": 0.21495679635292878, + "parent_plus_share": 0.21495679635292875, "parent_plus_by_reference_age": { "24_and_younger": 0.05361481308447381, "25_to_34": 0.00022032123488229902, "35_to_49": 0.03526511572758998, - "50_to_61": 0.4335550013453078, - "62_and_older": 0.7000193215077479 + "50_to_61": 0.43355500134530767, + "62_and_older": 0.7000193215077478 }, "by_head_age_band": { "24_and_younger": 0.2267174320421409, @@ -111,8 +111,8 @@ "mean_among_borrowers": 44483.09961323457, "median_among_borrowers": 25000.0, "total_household_weight": 122537091.8856833, - "federal_education_debt_total": 785628046119.6729, - "parent_plus_share_of_total": 0.21495679635292878 + "federal_education_debt_total": 785628046119.6727, + "parent_plus_share_of_total": 0.21495679635292875 }, "household_balance_bands": { "under_5k": 0.09597901426395242, @@ -273,7 +273,7 @@ "household_income" ], "share": 0.17281627661600854, - "parent_plus_share": 0.11558498718060217, + "parent_plus_share": 0.11558498718060221, "metrics_vs_scf": { "share_with_debt_error_pp": -0.8733291505995028, "mean_among_borrowers_error_pct": -4.349507836301458, @@ -288,13 +288,13 @@ }, "reason": "SCF MARRIED is married or living with a partner (X8023 in 1-2); A_MARITL 1-2 drops cohabiting partners.", "share": 0.14421786795177846, - "parent_plus_share": 0.23835459647602736, + "parent_plus_share": 0.2383545964760273, "parent_plus_by_reference_age": { "24_and_younger": 2.253143474595712e-05, "25_to_34": 0.0031437172991370397, - "35_to_49": 0.033462461180418936, - "50_to_61": 0.5087392931361828, - "62_and_older": 0.7514633083579594 + "35_to_49": 0.03346246118041894, + "50_to_61": 0.5087392931361827, + "62_and_older": 0.7514633083579593 }, "by_head_age_band": { "24_and_younger": 0.22671795358234773, @@ -329,7 +329,7 @@ "median_among_borrowers": 25300.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 790502796606.3756, - "parent_plus_share_of_total": 0.23835459647602736 + "parent_plus_share_of_total": 0.2383545964760273 }, "household_balance_bands": { "under_5k": 0.09724840853122779, @@ -490,7 +490,7 @@ "household_income" ], "share": 0.17281627661600854, - "parent_plus_share": 0.11558498718060217, + "parent_plus_share": 0.11558498718060221, "metrics_vs_scf": { "share_with_debt_error_pp": -0.8733291505995028, "mean_among_borrowers_error_pct": -4.349507836301458, @@ -505,12 +505,12 @@ }, "reason": "SCF KIDS counts children of the reference person or spouse/partner, any age, anywhere in the household listing.", "share": 0.1394955127190334, - "parent_plus_share": 0.17976252753958277, + "parent_plus_share": 0.1797625275395828, "parent_plus_by_reference_age": { "24_and_younger": 1.8950713199477208e-09, "25_to_34": 0.003974829123535003, "35_to_49": 0.023750250056890892, - "50_to_61": 0.4267091997855419, + "50_to_61": 0.4267091997855418, "62_and_older": 0.6189997224359128 }, "by_head_age_band": { @@ -545,8 +545,8 @@ "mean_among_borrowers": 43206.84231163398, "median_among_borrowers": 25300.0, "total_household_weight": 122537091.8856833, - "federal_education_debt_total": 738550734853.6542, - "parent_plus_share_of_total": 0.17976252753958277 + "federal_education_debt_total": 738550734853.654, + "parent_plus_share_of_total": 0.1797625275395828 }, "household_balance_bands": { "under_5k": 0.09939228853825689, @@ -707,7 +707,7 @@ "household_income" ], "share": 0.17281627661600854, - "parent_plus_share": 0.11558498718060217, + "parent_plus_share": 0.11558498718060221, "metrics_vs_scf": { "share_with_debt_error_pp": -0.8733291505995028, "mean_among_borrowers_error_pct": -4.349507836301458, @@ -722,12 +722,12 @@ }, "reason": "The SCF reference person comes from the economically dominant individual or couple (the man, or the older partner); the CPS householder anchors that unit, where the oldest adult can be a co-resident parent or roommate.", "share": 0.14160885581858304, - "parent_plus_share": 0.16635099149589067, + "parent_plus_share": 0.16635099149589064, "parent_plus_by_reference_age": { "24_and_younger": 0.00017157399333482214, "25_to_34": 0.005150069210217926, "35_to_49": 0.02057959141870075, - "50_to_61": 0.42901843742863854, + "50_to_61": 0.4290184374286386, "62_and_older": 0.6324187709904693 }, "by_head_age_band": { @@ -763,7 +763,7 @@ "median_among_borrowers": 25000.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 737509698707.2527, - "parent_plus_share_of_total": 0.16635099149589067 + "parent_plus_share_of_total": 0.16635099149589064 }, "household_balance_bands": { "under_5k": 0.09993330421288438, @@ -924,7 +924,7 @@ "household_income" ], "share": 0.17281627661600854, - "parent_plus_share": 0.11558498718060217, + "parent_plus_share": 0.11558498718060221, "metrics_vs_scf": { "share_with_debt_error_pp": -0.8733291505995028, "mean_among_borrowers_error_pct": -4.349507836301458, @@ -939,13 +939,13 @@ }, "reason": "SCF income covers the primary economic unit; the SPM unit is the CPS resource-sharing unit.", "share": 0.1422480075434781, - "parent_plus_share": 0.17924572328488167, + "parent_plus_share": 0.1792457232848817, "parent_plus_by_reference_age": { "24_and_younger": 2.0364004798094018e-08, "25_to_34": 0.0035008167958351234, - "35_to_49": 0.02945856222174237, - "50_to_61": 0.47869069913423223, - "62_and_older": 0.6254994464664675 + "35_to_49": 0.029458562221742368, + "50_to_61": 0.47869069913423207, + "62_and_older": 0.6254994464664674 }, "by_head_age_band": { "24_and_younger": 0.19738286662117155, @@ -980,7 +980,7 @@ "median_among_borrowers": 25000.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 741056653493.6942, - "parent_plus_share_of_total": 0.17924572328488167 + "parent_plus_share_of_total": 0.1792457232848817 }, "household_balance_bands": { "under_5k": 0.09952116824675251, @@ -1105,7 +1105,7 @@ "50_to_61": 0.19230144811764502, "62_and_older": 0.07158736116182915 }, - "graduate_share_of_own_original_principal": 0.536561414303497, + "graduate_share_of_own_original_principal": 0.5365614143034969, "ibr_new_borrower_share": 0.6249721194428659 } } @@ -1141,7 +1141,7 @@ "household_income" ], "share": 0.17281627661600854, - "parent_plus_share": 0.11558498718060217, + "parent_plus_share": 0.11558498718060221, "metrics_vs_scf": { "share_with_debt_error_pp": -0.8733291505995028, "mean_among_borrowers_error_pct": -4.349507836301458, @@ -1160,8 +1160,8 @@ "parent_plus_by_reference_age": { "24_and_younger": 1.352905945393419e-08, "25_to_34": 0.00013998491463271278, - "35_to_49": 0.019468440588935952, - "50_to_61": 0.39136120955142256, + "35_to_49": 0.019468440588935945, + "50_to_61": 0.3913612095514226, "62_and_older": 0.5990857762189746 }, "by_head_age_band": { @@ -1322,7 +1322,7 @@ "50_to_61": 0.19389979448680775, "62_and_older": 0.06907706577370382 }, - "graduate_share_of_own_original_principal": 0.497501055409478, + "graduate_share_of_own_original_principal": 0.4975010554094781, "ibr_new_borrower_share": 0.6165370510095016 } } @@ -1375,9 +1375,9 @@ "parent_plus_by_reference_age": { "24_and_younger": 4.3569134442118664e-10, "25_to_34": 0.0006982323042478812, - "35_to_49": 0.01612499187177073, - "50_to_61": 0.3707882978015445, - "62_and_older": 0.7327689853082329 + "35_to_49": 0.016124991871770737, + "50_to_61": 0.37078829780154465, + "62_and_older": 0.7327689853082328 }, "by_head_age_band": { "24_and_younger": 0.22636975285526706, @@ -1537,7 +1537,7 @@ "50_to_61": 0.2105771421991523, "62_and_older": 0.06126083438253632 }, - "graduate_share_of_own_original_principal": 0.4992132097267167, + "graduate_share_of_own_original_principal": 0.49921320972671657, "ibr_new_borrower_share": 0.5832347045593139 } } @@ -1584,12 +1584,12 @@ }, "reason": "As r1, and Social Security or pension income: 18% of host households at 25-34 have some against 6% of SCF families.", "share": 0.16413671000206656, - "parent_plus_share": 0.16758194869072, + "parent_plus_share": 0.16758194869071996, "parent_plus_by_reference_age": { "24_and_younger": 0.0009290910285421434, "25_to_34": 0.003457206986791982, - "35_to_49": 0.020444291263711122, - "50_to_61": 0.3174101032569371, + "35_to_49": 0.02044429126371113, + "50_to_61": 0.317410103256937, "62_and_older": 0.7753235949011145 }, "by_head_age_band": { @@ -1624,8 +1624,8 @@ "mean_among_borrowers": 42680.26879514089, "median_among_borrowers": 23000.0, "total_household_weight": 122537091.8856833, - "federal_education_debt_total": 858421208954.9308, - "parent_plus_share_of_total": 0.16758194869072 + "federal_education_debt_total": 858421208954.9309, + "parent_plus_share_of_total": 0.16758194869071996 }, "household_balance_bands": { "under_5k": 0.1179407532730876, @@ -1750,7 +1750,7 @@ "50_to_61": 0.20778027670943267, "62_and_older": 0.07240287305689257 }, - "graduate_share_of_own_original_principal": 0.4826934460430417, + "graduate_share_of_own_original_principal": 0.48269344604304165, "ibr_new_borrower_share": 0.6147006160485154 } } @@ -1797,13 +1797,13 @@ }, "reason": "As r2 with the household income unit, to separate the two changes.", "share": 0.16471154337166136, - "parent_plus_share": 0.21133541826981272, + "parent_plus_share": 0.21133541826981264, "parent_plus_by_reference_age": { "24_and_younger": 1.1813563767402388e-06, "25_to_34": 0.002878811061981838, - "35_to_49": 0.03437594638384092, - "50_to_61": 0.4760763618611841, - "62_and_older": 0.7759316514071254 + "35_to_49": 0.03437594638384093, + "50_to_61": 0.47607636186118407, + "62_and_older": 0.7759316514071253 }, "by_head_age_band": { "24_and_younger": 0.1992126713308297, @@ -1837,8 +1837,8 @@ "mean_among_borrowers": 42874.79958838091, "median_among_borrowers": 23000.0, "total_household_weight": 122537091.8856833, - "federal_education_debt_total": 865353807411.8154, - "parent_plus_share_of_total": 0.21133541826981272 + "federal_education_debt_total": 865353807411.8156, + "parent_plus_share_of_total": 0.21133541826981264 }, "household_balance_bands": { "under_5k": 0.11607469728946729, diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 956a7dcccf..f0caee74e4 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1254,8 +1254,6 @@ "is_married", "own_children_in_household", "employment_income", - "interest_dividend_income", - "social_security_pension_income", "household_income" ], "targets": [ @@ -1275,8 +1273,6 @@ "is_married", "own_children_in_household", "employment_income", - "interest_dividend_income", - "social_security_pension_income", "household_income", "federal_education_debt" ], @@ -1362,7 +1358,7 @@ "federal_parent_plus_loan_balance", "federal_student_loan_graduate_share" ], - "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family's federal share and holder split of loans 1-6. Whose education: code 1 (the reference person) and code 2 (spouse or partner) are each person's own federal_student_loan_balance; code 3 (child, with grandchild and other relative folded in by the public file) is the reference person's federal_parent_plus_loan_balance; codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the reference person's own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower is true when every federal loan the person holds was borrowed in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person's own federal loans that financed a master's, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all families on the eight shared predictors plus household income less capital gains; a second, fit on borrower families and conditioning on the total, draws the chained holder shares, graduate shares and new-borrower flags. The SCF-style reference person and the member named by its A_SPOUSE (else PECOHAB) pointer receive the balances; without one present the spouse's draw goes to the reference person. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals." + "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family's federal share and holder split of loans 1-6. Whose education: code 1 (the reference person) and code 2 (spouse or partner) are each person's own federal_student_loan_balance; code 3 (child, with grandchild and other relative folded in by the public file) is the reference person's federal_parent_plus_loan_balance; codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the reference person's own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower is true when every federal loan the person holds was borrowed in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person's own federal loans that financed a master's, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all families; a second, fit on borrower families and conditioning on the total, draws the chained holder shares, graduate shares and new-borrower flags. Both condition on the reference person's age, sex and race, a spouse or partner being present (SCF MARRIED counts partners), the children of the reference person or partner (SCF KIDS), and the wages and income less capital gains of the reference person, spouse or partner and their children (the CPS counterpart of the SCF primary economic unit). The shared interest-dividend and Social Security-pension predictors are left out: the CPS host carries tax-record imputations of both, so at reference ages 25-34 42% and 18% of host households report some against 16% and 6% of SCF families, and mapping them to the SCF distribution moves the host's borrower share most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). The reference person is the CPS householder or, with a spouse or partner present, the man of a mixed-sex couple or else the older partner; that person and the member named by its A_SPOUSE (else PECOHAB) pointer receive the balances, and without one present the spouse's draw goes to the reference person. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals." }, { "stage": "ssi_disability_criteria", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 4063aadbe5..1487763de7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -767,7 +767,10 @@ from microcosm.build.us_runtime.scf_education_debt import ( SCF_2025_FULL_EXTRACT, SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS, SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, SCF_EDUCATION_DEBT_VINTAGE, SCF_WHOSE_EDUCATION_HOLDER, US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS, @@ -1596,7 +1599,10 @@ "with_us_scf_auto_loan_inputs", "SCF_2025_FULL_EXTRACT", "SCF_2025_SUMMARY_EXTRACT", + "SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS", + "SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS", "SCF_EDUCATION_DEBT_PREDICTORS", + "SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION", "SCF_EDUCATION_DEBT_VINTAGE", "SCF_WHOSE_EDUCATION_HOLDER", "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py index e09789e140..e86e7480a7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -58,23 +58,47 @@ weights the SAVE rates. **Imputation.** Two regime-gated weighted QRFs (:class:`microcosm.fit.QRF`) -condition on the eight shared SCF predictors plus household income excluding -capital gains (:data:`SCF_EDUCATION_DEBT_PREDICTORS`). The first draws each -recipient household's federal education debt from every SCF family. The -second, fit on borrower families only and conditioning also on that total, +condition on the same predictors (:data:`SCF_EDUCATION_DEBT_PREDICTORS`): the +reference person's age, sex and race, whether a spouse or partner is present, +the number of children, wages, and income excluding capital gains. The first +draws each recipient household's federal education debt from every SCF family. +The second, fit on borrower families only and conditioning also on that total, draws the chained composition: the head's and spouse's shares of the total, their graduate shares, and their new-borrower flags. Shares are clipped to [0, 1] and rescaled when the two own shares exceed one; the remainder is the head's Parent PLUS balance. -**Recipients.** The household's SCF-style reference person -(:func:`scf_auto_loans._scf_reference_person_mask`) carries the reference -person's demographics and the household-summed incomes, as for auto loans. The -spouse is the member whose line number is the reference person's ``A_SPOUSE``, -or else its ``PECOHAB`` partner when that column is present. A household drawn -with a spouse's balance but no spouse or partner present gives that balance to -the reference person, whose graduate share becomes the balance-weighted mean -of the two and whose new-borrower flag requires both. Everyone else gets zero. +Two of the shared SCF predictors are left out +(:data:`SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS` keeps the full list for the +diagnostics). Interest and dividends, and Social Security and pensions, are +measured differently on the two sides: the CPS host carries tax-record +imputations, the SCF survey reports. At reference ages 25-34, 42% of host +households have interest or dividends against 16% of SCF families, and 18% +have Social Security or pension income against 6%. A QRF reads equal values +as equal positions, so the host's older-looking households drew less debt: +mapping these two predictors to their SCF distribution at equal rank moves the +host's borrower share most of any predictor +(``experiments/receipts/1073-predictor-shift-diagnosis.json``). Wages and +total income stay in, because the issue asks for income and they carry the +income gradient. + +**Recipients** (:data:`SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION`). The CPS side +builds each predictor the way the SCF defines it. The reference person is the +CPS householder or, when a spouse or cohabiting partner is present, the man of +a mixed-sex couple or else the older partner: the SCF's ordering of its +economically dominant couple. ``is_married`` counts a cohabiting partner, as +SCF ``MARRIED`` does, and children count those of the reference person or +partner, as SCF ``KIDS`` does. Incomes sum over the reference person, spouse or +partner and their children: the SCF primary economic unit, which leaves out a +financially independent co-resident relative or roommate. The spouse is the +member whose line number is the reference person's ``A_SPOUSE``, or else its +``PECOHAB`` partner when that column is present. A household drawn with a +spouse's balance but no spouse or partner present gives that balance to the +reference person, whose graduate share becomes the balance-weighted mean of +the two and whose new-borrower flag requires both. Everyone else gets zero. +:data:`SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION` keeps the first receipt's +construction (oldest adult, ``A_MARITL`` marriage, own children, whole +household) for its before/after table. **Known limits.** The SCF records the debts of the primary economic unit, so a financially independent adult living in someone else's household is missed, @@ -113,6 +137,8 @@ "SCF_2025_SUMMARY_EXTRACT", "SCF_EDUCATION_DEBT_BALANCE_BANDS", "SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS", + "SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS", + "SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS", "SCF_EDUCATION_DEBT_COMPOSITION_TARGETS", "SCF_EDUCATION_DEBT_PREDICTORS", "SCF_EDUCATION_DEBT_TOTAL_TARGET", @@ -284,12 +310,37 @@ def zip_name(self) -> str: "head_is_ibr_new_borrower", "spouse_is_ibr_new_borrower", ) -#: The shared SCF predictors plus household income excluding capital gains. +#: Every predictor both sides build: the shared SCF predictors plus income +#: excluding capital gains. The diagnostics and ablations compare all of them. #: Educational attainment joins once microcosm#297 carries it on the CPS side. -SCF_EDUCATION_DEBT_PREDICTORS: tuple[str, ...] = ( +SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS: tuple[str, ...] = ( *SCF_WEALTH_PREDICTORS, "household_income", ) +#: Candidate predictors the QRFs leave out, each with its reason. Mapping each +#: to its SCF distribution at equal rank within five-year reference-person age +#: bins moves the host's borrower share from 14.2% to 15.8% (interest and +#: dividends) and 14.7% (Social Security and pensions); all candidates together +#: reach 17.3%, the SCF hold-out's level +#: (experiments/receipts/1073-predictor-shift-diagnosis.json). +SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS: Mapping[str, str] = { + "interest_dividend_income": ( + "CPS interest and dividends are tax-record imputations: 42% of host " + "households at reference ages 25-34 have some, against 16% of SCF " + "families." + ), + "social_security_pension_income": ( + "CPS Social Security and pension income is likewise imputed: 18% of " + "host households at reference ages 25-34 have some, against 6% of SCF " + "families." + ), +} +#: The predictors both QRFs condition on. +SCF_EDUCATION_DEBT_PREDICTORS: tuple[str, ...] = tuple( + predictor + for predictor in SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS + if predictor not in SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS +) US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS: tuple[str, ...] = ( "federal_student_loan_balance", @@ -788,7 +839,7 @@ class RecipientDefinition: reference_person: str = "householder_couple" partnered: str = "spouse_or_partner" children: str = "of_reference_or_partner" - income_unit: str = "household" + income_unit: str = "reference_couple_and_children" def __post_init__(self) -> None: for value, allowed, name in ( @@ -1068,7 +1119,7 @@ def _recipients( features[column] = sums features.index = household.index return _Recipients( - features=features.loc[:, list(SCF_EDUCATION_DEBT_PREDICTORS)], + features=features.loc[:, list(SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS)], head_position=head_position, spouse_position=spouse_position, ) @@ -1376,18 +1427,32 @@ def us_scf_education_debt_stage_spec() -> SourceStageSpec: f"{_SCF_STAGE_NAME!r} SCF {SCF_EDUCATION_DEBT_VINTAGE} artifact pin " f"drift for {public_file.member}: {drift}." ) - fits = [ - operation - for operation in spec.operations - if operation.kind == "fit_weighted_qrf" - and operation.parameters.get("donor") == "scf_2025_education_debt" - ] - if len(fits) != 1 or tuple(fits[0].parameters.get("predictors", ())) != ( - SCF_EDUCATION_DEBT_PREDICTORS - ): - raise ValueError( - f"{_SCF_STAGE_NAME!r} education-debt QRF predictors have drifted." - ) + expected_fits = { + "scf_2025_education_debt": ( + SCF_EDUCATION_DEBT_PREDICTORS, + (SCF_EDUCATION_DEBT_TOTAL_TARGET,), + ), + "scf_2025_education_debt_borrowers": ( + (*SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_TOTAL_TARGET), + SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + ), + } + for donor, (predictors, targets) in expected_fits.items(): + fits = [ + operation + for operation in spec.operations + if operation.kind == "fit_weighted_qrf" + and operation.parameters.get("donor") == donor + ] + if ( + len(fits) != 1 + or tuple(fits[0].parameters.get("predictors", ())) != predictors + or tuple(fits[0].parameters.get("targets", ())) != targets + ): + raise ValueError( + f"{_SCF_STAGE_NAME!r} education-debt QRF {donor!r} predictors or " + "targets have drifted." + ) return spec diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index c2e864b912..f0891cedf4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720", + "sha256": "775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py index 249c273984..6c38e30fcc 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py @@ -303,6 +303,42 @@ def test_stage_declares_the_person_outputs_and_pinned_2025_files() -> None: assert spec.artifacts[0]["vintage"] == "2022" assert spec.artifacts[1]["vintage"] == "2022" assert spec.artifacts[2]["member"] == "pu2023.csv" + # Both QRFs condition on the production predictors; the borrower fit adds + # the drawn total. + fits = { + operation.parameters["donor"]: operation.parameters + for operation in spec.operations + if operation.kind == "fit_weighted_qrf" + and str(operation.parameters.get("donor", "")).startswith("scf_2025") + } + assert tuple(fits["scf_2025_education_debt"]["predictors"]) == ( + SCF_EDUCATION_DEBT_PREDICTORS + ) + assert tuple(fits["scf_2025_education_debt_borrowers"]["predictors"]) == ( + *SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + ) + + +def test_production_predictors_drop_only_the_reviewed_exclusions() -> None: + candidates = education_debt.SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS + excluded = education_debt.SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS + assert set(excluded) == { + "interest_dividend_income", + "social_security_pension_income", + } + assert all(reason.strip() for reason in excluded.values()) + assert SCF_EDUCATION_DEBT_PREDICTORS == tuple( + predictor for predictor in candidates if predictor not in excluded + ) + assert education_debt.SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION == ( + education_debt.RecipientDefinition( + reference_person="householder_couple", + partnered="spouse_or_partner", + children="of_reference_or_partner", + income_unit="reference_couple_and_children", + ) + ) def _zip_with(member: str, payload: bytes) -> bytes: @@ -746,7 +782,12 @@ def test_default_definition_follows_the_scf_definitions() -> None: assert features["is_married"].tolist() == [1.0, 1.0] # Household 1 counts the partner's child; the mother is no one's child. assert features["own_children_in_household"].tolist() == [1.0, 0.0] - assert features["employment_income"].tolist() == [120_000.0, 0.0] + # Income sums over the couple and their children; the mother's wages are + # outside the SCF primary economic unit. + assert features["employment_income"].tolist() == [90_000.0, 0.0] + assert list(features.columns) == list( + education_debt.SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS + ) def test_first_receipt_definition_reproduces_the_oldest_adult_rule() -> None: diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 10c3280084..c7b15e3a86 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720" + "775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 89d025cebf..c2d6098fcf 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720" + "775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From 5d047f687f898a7b9d22ebd7b4593f7037626c17 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Tue, 20 Oct 2026 12:00:38 -0400 Subject: [PATCH 05/17] Regenerate the US stage spec for the SCF-comparable education-debt predictors (#1073) Output of tools/generate_us_bundle_from_constants.py (run on Modal at b9f3d6719, run 20261020T154855Z-b9f3d6719): both education-debt QRFs drop interest_dividend_income and social_security_pension_income, the stage notes follow the module, and the stage-asset digest moves. The Modal experiment runner now strips the Modal runtime's PYTHONPATH from step environments; inheriting it put Modal's own packages beside the venv's and the worker-identity check refused the duplicate idna. Co-Authored-By: Claude Opus 5.5 (1M context) --- experiments/modal_1073.py | 44 +++++++++++++------ .../src/microcosm/build/us/spec/sources.yaml | 24 +++++----- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/experiments/modal_1073.py b/experiments/modal_1073.py index c64ccd1cb2..16eefd55aa 100644 --- a/experiments/modal_1073.py +++ b/experiments/modal_1073.py @@ -203,6 +203,22 @@ def _verified(path: Path, sha256: str) -> bool: return path.exists() and _sha256(path) == sha256 +def _clean_env(**overrides: str) -> dict[str, str]: + """The container environment without the Modal runtime's Python path. + + Modal puts its own client and dependencies (under /pkg) on PYTHONPATH. A + subprocess that inherits it sees those packages beside the venv's, and the + build's worker-identity check refuses the duplicates (e.g. idna). + """ + env = { + key: value + for key, value in os.environ.items() + if key not in {"PYTHONPATH", "PYTHONHOME", "PYTHONSTARTUP"} + } + env.update(overrides) + return env + + def _checkout(commit: str) -> None: WORK.mkdir(parents=True, exist_ok=True) run = lambda *args: subprocess.run(args, cwd=WORK, check=True) # noqa: E731 @@ -219,13 +235,15 @@ def _checkout(commit: str) -> None: ).stdout.strip() if head != commit: raise RuntimeError(f"checked out {head}, expected {commit}") - env = os.environ | { - "UV_PROJECT_ENVIRONMENT": str(VENV), - # A Modal volume cannot host uv's cache: uv persists temporary files - # by atomic rename, which the volume refuses. - "UV_CACHE_DIR": "/root/.cache/uv", - "UV_LINK_MODE": "copy", - } + env = _clean_env( + **{ + "UV_PROJECT_ENVIRONMENT": str(VENV), + # A Modal volume cannot host uv's cache: uv persists temporary files + # by atomic rename, which the volume refuses. + "UV_CACHE_DIR": "/root/.cache/uv", + "UV_LINK_MODE": "copy", + } + ) subprocess.run( ["uv", "sync", "--all-packages", "--extra", "us", "--frozen"], cwd=WORK, @@ -351,12 +369,12 @@ def run_steps(commit: str, steps: list[str], run_id: str) -> dict: "run_id": run_id, } threads = str(int(CPU)) - env = os.environ | { - "PYTHONUNBUFFERED": "1", - "OMP_NUM_THREADS": threads, - "MKL_NUM_THREADS": threads, - "OPENBLAS_NUM_THREADS": threads, - } + env = _clean_env( + PYTHONUNBUFFERED="1", + OMP_NUM_THREADS=threads, + MKL_NUM_THREADS=threads, + OPENBLAS_NUM_THREADS=threads, + ) out = CACHE / "results" / run_id out.mkdir(parents=True, exist_ok=True) log = { diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 5711f25977..ecd774c7ec 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: c4aa84a18c826543ba4416ba3b1ef9ce30c5770a7d942e601685e04a3d4e2720 + sha256: 775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597 stage_manifest: version: 1 country: us @@ -1290,8 +1290,6 @@ stages: - is_married - own_children_in_household - employment_income - - interest_dividend_income - - social_security_pension_income - household_income targets: - federal_education_debt @@ -1307,8 +1305,6 @@ stages: - is_married - own_children_in_household - employment_income - - interest_dividend_income - - social_security_pension_income - household_income - federal_education_debt targets: @@ -1412,11 +1408,19 @@ stages: in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person''s own federal loans that financed a master''s, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all - families on the eight shared predictors plus household income less capital gains; a second, fit on - borrower families and conditioning on the total, draws the chained holder shares, graduate shares - and new-borrower flags. The SCF-style reference person and the member named by its A_SPOUSE (else - PECOHAB) pointer receive the balances; without one present the spouse''s draw goes to the reference - person. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals.' + families; a second, fit on borrower families and conditioning on the total, draws the chained holder + shares, graduate shares and new-borrower flags. Both condition on the reference person''s age, sex + and race, a spouse or partner being present (SCF MARRIED counts partners), the children of the reference + person or partner (SCF KIDS), and the wages and income less capital gains of the reference person, + spouse or partner and their children (the CPS counterpart of the SCF primary economic unit). The shared + interest-dividend and Social Security-pension predictors are left out: the CPS host carries tax-record + imputations of both, so at reference ages 25-34 42% and 18% of host households report some against + 16% and 6% of SCF families, and mapping them to the SCF distribution moves the host''s borrower share + most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). The reference person + is the CPS householder or, with a spouse or partner present, the man of a mixed-sex couple or else + the older partner; that person and the member named by its A_SPOUSE (else PECOHAB) pointer receive + the balances, and without one present the spouse''s draw goes to the reference person. Balances are + 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals.' - stage: ssi_disability_criteria survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html From e35c4b0f911883585c150d2d6417dc01da2bb9d9 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Tue, 20 Oct 2026 12:22:23 -0400 Subject: [PATCH 06/17] Make the spouse-routing test independent of draw chance (#1073) test_a_spouse_draw_without_a_spouse_moves_to_the_reference_person checks where a spouse's draw lands. With the synthetic donor's random incidence and a 30-household frame, the SCF-comparable predictor set draws no debt for any couple on 3 of 20 seeds (seed 11 among them), while the routing assertion holds on all 20 for both predictor sets (Modal probe run 20261020T161732Z-5d047f687). Every donor family now borrows in this test. The Modal runner gains --script for an uncommitted local probe, recorded by sha256 in the run log. Co-Authored-By: Claude Opus 5.5 (1M context) --- experiments/modal_1073.py | 22 ++++++++++++++----- .../us/test_us_scf_education_debt.py | 7 ++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/experiments/modal_1073.py b/experiments/modal_1073.py index 16eefd55aa..c19a09d446 100644 --- a/experiments/modal_1073.py +++ b/experiments/modal_1073.py @@ -358,7 +358,9 @@ def preflight_remote(commit: str) -> dict: @app.function( image=image, cpu=CPU, memory=MEMORY_MB, timeout=4 * 3600, volumes={"/cache": cache} ) -def run_steps(commit: str, steps: list[str], run_id: str) -> dict: +def run_steps( + commit: str, steps: list[str], run_id: str, script_text: str = "" +) -> dict: import resource # Unix-only; the local entrypoint runs on Windows too started = datetime.now(UTC) @@ -388,7 +390,14 @@ def run_steps(commit: str, steps: list[str], run_id: str) -> dict: } failed = False for step in steps: - commands = [_expand(command, fields) for command in STEPS[step]] + if step == "script": + # An uncommitted local probe passed by --script; it runs in the + # clone like any step, and the run log records its text. + (WORK / "_modal_probe.py").write_text(script_text, encoding="utf-8") + log["script_sha256"] = hashlib.sha256(script_text.encode()).hexdigest() + commands = [[fields["python"], "_modal_probe.py"]] + else: + commands = [_expand(command, fields) for command in STEPS[step]] before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss t0 = time.time() returncodes = [] @@ -450,15 +459,18 @@ def preflight(commit: str) -> None: @app.local_entrypoint() -def main(commit: str, steps: str = "tests,shift,explore") -> None: +def main(commit: str, steps: str = "tests,shift,explore", script: str = "") -> None: _check_commit(commit) names = [s for s in steps.split(",") if s] - unknown = sorted(set(names) - set(STEPS)) + script_text = Path(script).read_text(encoding="utf-8") if script else "" + if script and "script" not in names: + names.append("script") + unknown = sorted(set(names) - set(STEPS) - {"script"}) if unknown: raise SystemExit(f"unknown steps: {unknown}") run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + commit[:9] print(f"run_id={run_id}") - print(json.dumps(run_steps.remote(commit, names, run_id), indent=2)) + print(json.dumps(run_steps.remote(commit, names, run_id, script_text), indent=2)) @app.local_entrypoint() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py index 6c38e30fcc..9e6dc5c416 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py @@ -556,6 +556,13 @@ def test_imputation_reaches_only_reference_persons_and_their_spouses() -> None: def test_a_spouse_draw_without_a_spouse_moves_to_the_reference_person() -> None: frame = _us_frame(_person_rows(30)) donor = _ready_donor() + # Every donor family borrows, so every household draws a positive total and + # the test checks only where the spouse's draw lands. With the synthetic + # donor's random incidence, a 30-household frame otherwise draws no debt for + # any couple on about 2 seeds in 20. + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = np.maximum( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], 1_000.0 + ) donor["head_own_share"] = np.where(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 0, 0) donor["spouse_own_share"] = np.where( donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 1.0, 0.0 From b1a461f196910eb41e157b83773c6d87d91f2ca2 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Wed, 21 Oct 2026 10:53:22 -0400 Subject: [PATCH 07/17] Collect the e35c4b0f receipts, add a Modal launch entrypoint, bind the loan register entries (#1073) - experiments/1073-scf-education-debt-receipts.md and its JSON are the receipt of Modal run 20261021T133110Z-e35c4b0f9 at e35c4b0f9 (build, first-receipt and all-candidates settings on the published default). - experiments/modal_1073.py gains `launch`, which spawns the run and exits, so a stopped local client cannot cancel it, and `--script` for an uncommitted probe. - test_support/microcosm_data/stored_inputs.py lists the four federal student loan inputs among the producer-bound register entries, next to the parent ids: no file examined for #1026 stores them, and test_us_stored_input_register.py binds them to their live producer. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-education-debt-receipts.md | 171 ++- experiments/modal_1073.py | 25 +- .../receipts/1073-scf-education-debt-fit.json | 1041 +++++++++++++++-- test_support/microcosm_data/stored_inputs.py | 15 +- 4 files changed, 1057 insertions(+), 195 deletions(-) diff --git a/experiments/1073-scf-education-debt-receipts.md b/experiments/1073-scf-education-debt-receipts.md index 4abf2c176e..45603c76a0 100644 --- a/experiments/1073-scf-education-debt-receipts.md +++ b/experiments/1073-scf-education-debt-receipts.md @@ -1,8 +1,10 @@ # microcosm#1073 receipts: SCF 2025 federal student loan balances -Generated 2026-10-12 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`. Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. +Generated 2026-10-21 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`, commit `e35c4b0f911883585c150d2d6417dc01da2bb9d9`, Modal run `20261021T133110Z-e35c4b0f9` (`experiments/modal_1073.py`). Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. -Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5). Household figures group by the reference person's age and race and by weighted quintiles of household income less capital gains, the income concept the imputation conditions on. Borrower figures count people with a federal balance (own plus Parent PLUS), in Federal Student Aid's balance and age bands; SCF borrowers are the reference person and spouse. Nothing here is a gate or a calibration target. +Host columns impute the published default `populace-us-2024-spm-20260915` at its calibrated weights under three settings: the first receipt's construction (oldest adult, `A_MARITL` marriage, own children, whole-household income, every candidate predictor); the SCF-matching codings with every candidate predictor; and the build, which also leaves out interest and dividends and Social Security and pensions. The cause and the evidence are in the stage notes and `experiments/receipts/1073-predictor-shift-diagnosis.json`. + +Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5). Household figures group by the reference person's age and race and by weighted quintiles of income less capital gains of the setting's income unit, the income concept that setting conditions on. Borrower figures count people with a federal balance (own plus Parent PLUS), in Federal Student Aid's balance and age bands; SCF borrowers are the reference person and spouse. Nothing here is a gate or a calibration target. ## Source checks @@ -30,100 +32,100 @@ Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5 ## Fit -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| Households with federal education debt | 18.2% | 17.3% | 14.4% | -| Mean among them | $41,052 | $39,267 | $44,483 | -| Median among them | $25,000 | $23,000 | $25,000 | -| Weighted federal education debt | $1.006T | $0.916T | $0.786T | -| Parent PLUS share of the total | 13.8% | 11.6% | 21.5% | -| Borrowers (people) | 27.2M | 24.7M | 18.5M | -| Mean balance per borrower | $36,934 | $37,030 | $42,554 | -| Median balance per borrower | $21,000 | $22,000 | $24,000 | -| IBR new borrowers among borrowers | 59.2% | 59.2% | 60.6% | -| Graduate share of own original principal | 47.7% | 46.2% | 53.6% | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| Households with federal education debt | 18.2% | 17.9% | 14.4% | 14.5% | 16.4% | +| Mean among them | $41,052 | $38,130 | $44,483 | $42,174 | $42,680 | +| Median among them | $25,000 | $23,000 | $25,000 | $25,000 | $23,000 | +| Weighted federal education debt | $1.006T | $0.919T | $0.786T | $0.747T | $0.858T | +| Parent PLUS share of the total | 13.8% | 12.2% | 21.5% | 15.5% | 16.8% | +| Borrowers (people) | 27.2M | 25.7M | 18.5M | 18.5M | 21.2M | +| Mean balance per borrower | $36,934 | $35,733 | $42,554 | $40,307 | $40,447 | +| Median balance per borrower | $21,000 | $21,000 | $24,000 | $24,000 | $22,000 | +| IBR new borrowers among borrowers | 59.2% | 59.3% | 60.6% | 61.7% | 61.5% | +| Graduate share of own original principal | 47.7% | 45.0% | 53.6% | 49.8% | 48.3% | ### Borrowers by FSA balance band -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| under_5k | 11.9% | 10.8% | 10.4% | -| 5k_to_10k | 15.5% | 15.2% | 16.7% | -| 10k_to_20k | 18.6% | 18.3% | 15.5% | -| 20k_to_40k | 22.3% | 24.0% | 23.0% | -| 40k_to_60k | 12.8% | 12.2% | 10.4% | -| 60k_to_80k | 8.1% | 9.4% | 9.6% | -| 80k_to_100k | 3.7% | 3.5% | 5.1% | -| 100k_to_200k | 4.9% | 4.8% | 6.4% | -| 200k_plus | 2.2% | 1.9% | 2.9% | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| under_5k | 11.9% | 10.2% | 10.4% | 11.8% | 12.5% | +| 5k_to_10k | 15.5% | 16.6% | 16.7% | 15.4% | 13.6% | +| 10k_to_20k | 18.6% | 18.7% | 15.5% | 16.1% | 19.0% | +| 20k_to_40k | 22.3% | 23.6% | 23.0% | 23.8% | 23.5% | +| 40k_to_60k | 12.8% | 12.0% | 10.4% | 11.5% | 10.8% | +| 60k_to_80k | 8.1% | 8.6% | 9.6% | 9.1% | 8.7% | +| 80k_to_100k | 3.7% | 3.9% | 5.1% | 3.9% | 3.2% | +| 100k_to_200k | 4.9% | 4.7% | 6.4% | 5.5% | 5.6% | +| 200k_plus | 2.2% | 1.6% | 2.9% | 2.9% | 3.1% | ### Borrowers by FSA age band -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| 24_and_younger | 5.7% | 6.1% | 3.8% | -| 25_to_34 | 33.1% | 33.3% | 26.6% | -| 35_to_49 | 38.1% | 38.4% | 39.8% | -| 50_to_61 | 16.2% | 15.8% | 21.1% | -| 62_and_older | 6.9% | 6.4% | 8.7% | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 24_and_younger | 5.7% | 6.5% | 3.8% | 5.5% | 5.0% | +| 25_to_34 | 33.1% | 33.9% | 26.6% | 30.9% | 29.2% | +| 35_to_49 | 38.1% | 38.0% | 39.8% | 37.3% | 37.8% | +| 50_to_61 | 16.2% | 15.1% | 21.1% | 19.4% | 20.8% | +| 62_and_older | 6.9% | 6.6% | 8.7% | 6.9% | 7.2% | ### Households with federal education debt, by age of reference person -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| 24_and_younger | 22.2% | 24.4% | 22.7% | -| 25_to_34 | 36.2% | 34.7% | 26.7% | -| 35_to_49 | 27.2% | 27.2% | 22.3% | -| 50_to_61 | 16.7% | 14.5% | 14.2% | -| 62_and_older | 4.2% | 3.3% | 3.7% | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 24_and_younger | 22.2% | 25.0% | 22.7% | 21.1% | 21.8% | +| 25_to_34 | 36.2% | 36.9% | 26.7% | 27.1% | 28.1% | +| 35_to_49 | 27.2% | 27.4% | 22.3% | 20.9% | 24.4% | +| 50_to_61 | 16.7% | 14.8% | 14.2% | 13.3% | 16.4% | +| 62_and_older | 4.2% | 3.5% | 3.7% | 3.1% | 3.7% | Median among borrowers: -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| 24_and_younger | $10,000 | $11,000 | $6,000 | -| 25_to_34 | $24,900 | $24,600 | $20,200 | -| 35_to_49 | $25,000 | $22,000 | $25,670 | -| 50_to_61 | $28,000 | $28,000 | $29,000 | -| 62_and_older | $26,000 | $28,000 | $38,000 | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 24_and_younger | $10,000 | $14,000 | $6,000 | $14,000 | $12,662 | +| 25_to_34 | $24,900 | $23,000 | $20,200 | $21,600 | $20,100 | +| 35_to_49 | $25,000 | $22,900 | $25,670 | $25,000 | $23,000 | +| 50_to_61 | $28,000 | $27,000 | $29,000 | $30,000 | $28,742 | +| 62_and_older | $26,000 | $26,997 | $38,000 | $26,000 | $34,000 | ### Households with federal education debt, by race (`racecl4`) -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| white_non_hispanic | 17.2% | 15.9% | 13.3% | -| black_non_hispanic | 24.3% | 22.5% | 22.8% | -| hispanic | 14.1% | 16.1% | 12.3% | -| other_or_multiple | 19.5% | 19.2% | 15.3% | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| white_non_hispanic | 17.2% | 16.7% | 13.3% | 13.3% | 15.3% | +| black_non_hispanic | 24.3% | 23.2% | 22.8% | 22.8% | 24.9% | +| hispanic | 14.1% | 16.2% | 12.3% | 11.8% | 13.8% | +| other_or_multiple | 19.5% | 18.9% | 15.3% | 17.0% | 18.4% | Median among borrowers: -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| white_non_hispanic | $23,000 | $24,000 | $25,000 | -| black_non_hispanic | $30,000 | $24,600 | $28,000 | -| hispanic | $23,000 | $20,000 | $24,764 | -| other_or_multiple | $20,000 | $22,000 | $24,000 | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| white_non_hispanic | $23,000 | $23,000 | $25,000 | $25,000 | $23,000 | +| black_non_hispanic | $30,000 | $25,000 | $28,000 | $28,000 | $28,000 | +| hispanic | $23,000 | $20,000 | $24,764 | $23,000 | $19,200 | +| other_or_multiple | $20,000 | $21,000 | $24,000 | $22,000 | $18,000 | ### Households with federal education debt, by income quintile -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| 1 | 11.4% | 10.3% | 12.7% | -| 2 | 17.1% | 17.1% | 13.6% | -| 3 | 17.8% | 17.9% | 13.4% | -| 4 | 26.6% | 25.5% | 20.0% | -| 5 | 17.9% | 15.7% | 12.4% | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 1 | 11.4% | 10.4% | 12.7% | 13.6% | 14.3% | +| 2 | 17.1% | 17.8% | 13.6% | 14.6% | 15.8% | +| 3 | 17.8% | 18.0% | 13.4% | 13.0% | 14.7% | +| 4 | 26.6% | 26.3% | 20.0% | 19.2% | 23.1% | +| 5 | 17.9% | 16.8% | 12.4% | 11.8% | 14.2% | Median among borrowers: -| | SCF 2025 | SCF hold-out draw | Imputed onto populace-us-2024-spm-20260915 | -|---|---:|---:|---:| -| 1 | $19,200 | $19,200 | $20,000 | -| 2 | $22,000 | $22,000 | $26,000 | -| 3 | $21,000 | $23,000 | $23,000 | -| 4 | $28,000 | $25,000 | $25,000 | -| 5 | $28,000 | $26,000 | $28,000 | +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 1 | $19,200 | $19,200 | $20,000 | $20,000 | $20,000 | +| 2 | $22,000 | $21,000 | $26,000 | $23,000 | $20,000 | +| 3 | $21,000 | $21,000 | $23,000 | $23,000 | $23,000 | +| 4 | $28,000 | $25,000 | $25,000 | $26,000 | $26,000 | +| 5 | $28,000 | $28,000 | $28,000 | $26,814 | $23,000 | ## Predictor ablation (five-fold hold-out on SCF families) @@ -133,21 +135,18 @@ Each variant refits both QRFs. Errors are drawn minus observed; TVD is total var |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| | shared_8_without_age | -0.250 | -1.737 | -4.000 | 0.022 | 4.812 | 0.648 | 0.672 | 0.028 | 0.107 | 0.154 | | shared_8 | -0.532 | -4.092 | -8.000 | 0.041 | 1.193 | 1.078 | 0.876 | 0.037 | 0.016 | 0.173 | -| shared_8_plus_household_income (build) | -0.873 | -4.350 | -8.000 | 0.037 | 1.351 | 1.359 | 0.923 | 0.030 | 0.009 | 0.175 | +| shared_8_plus_household_income | -0.873 | -4.350 | -8.000 | 0.037 | 1.351 | 1.359 | 0.923 | 0.030 | 0.009 | 0.175 | | plus_household_income_and_education | -1.474 | -5.567 | -4.816 | 0.040 | 1.321 | 2.067 | 1.474 | 0.050 | 0.023 | 0.208 | +| build_without_interest_and_ss_pension (build) | -0.304 | -7.119 | -8.000 | 0.032 | 1.260 | 1.069 | 0.675 | 0.033 | 0.015 | 0.168 | +| build_plus_education | -0.920 | -7.087 | -8.000 | 0.028 | 1.749 | 1.789 | 0.920 | 0.029 | 0.031 | 0.207 | -## Host fit +## Host fit, before and after -Imputed with build seed 0 onto `populace-us-2024-spm-20260915` (57,240 households, sha256 `6496cc4393d4...`) at its calibrated household weights; the published weights were calibrated without these columns. Signal gate: passed. +Imputed with build seed 0 onto `populace-us-2024-spm-20260915` (57,240 households, sha256 `6496cc4393d4...`) at its calibrated household weights; the published weights were calibrated without these columns. Build signal gate: passed. Errors are imputed minus SCF 2025. -| Metric (imputed minus SCF 2025) | Value | -|---|---:| -| share_with_debt_error_pp | -3.742 | -| mean_among_borrowers_error_pct | 8.357 | -| median_among_borrowers_error_pct | 0.000 | -| household_balance_band_tvd | 0.062 | -| head_age_band_share_mae_pp | 3.554 | -| race_share_mae_pp | 2.834 | -| income_quintile_share_mae_pp | 4.252 | -| borrower_balance_band_tvd | 0.070 | -| borrower_age_band_tvd | 0.085 | +| Setting | Households with debt | Parent PLUS share | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| First receipt | 14.4% | 21.5% | -3.742 | 8.357 | 0.000 | 0.062 | 3.554 | 2.834 | 4.252 | 0.070 | 0.085 | +| SCF codings, all predictors | 14.5% | 15.5% | -3.701 | 2.732 | 0.000 | 0.040 | 4.170 | 2.570 | 4.568 | 0.040 | 0.032 | +| Build | 16.4% | 16.8% | -1.741 | 3.966 | -8.000 | 0.046 | 2.391 | 0.942 | 2.866 | 0.045 | 0.050 | +| SCF 2025 | 18.2% | 13.8% | | | | | | | | | | diff --git a/experiments/modal_1073.py b/experiments/modal_1073.py index c19a09d446..c4a4b575b6 100644 --- a/experiments/modal_1073.py +++ b/experiments/modal_1073.py @@ -12,7 +12,7 @@ export PYTHONIOENCODING=utf-8 PYTHONUTF8=1 modal run experiments/modal_1073.py::preflight --commit - modal run --detach experiments/modal_1073.py::main --commit --steps tests,shift,explore + modal run --detach experiments/modal_1073.py::launch --commit --steps tests,shift,explore modal run experiments/modal_1073.py::collect --run-id Steps (see STEPS; each runs one or more commands, stopping at the first that @@ -473,6 +473,29 @@ def main(commit: str, steps: str = "tests,shift,explore", script: str = "") -> N print(json.dumps(run_steps.remote(commit, names, run_id, script_text), indent=2)) +@app.local_entrypoint() +def launch(commit: str, steps: str = "tests,shift,explore", script: str = "") -> None: + """Spawn the run and exit at once (use with ``modal run --detach``). + + ``main`` keeps a local client attached for the whole run, and stopping that + client (for example when the machine runs low on memory) cancels the + remote input even under --detach. ``launch`` returns as soon as the call is + spawned, so nothing local can cancel it; ``collect`` fetches the results. + """ + _check_commit(commit) + names = [s for s in steps.split(",") if s] + script_text = Path(script).read_text(encoding="utf-8") if script else "" + if script and "script" not in names: + names.append("script") + unknown = sorted(set(names) - set(STEPS) - {"script"}) + if unknown: + raise SystemExit(f"unknown steps: {unknown}") + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + commit[:9] + call = run_steps.spawn(commit, names, run_id, script_text) + print(f"run_id={run_id}") + print(f"function_call_id={call.object_id}") + + @app.local_entrypoint() def collect(run_id: str, logs_dir: str = "") -> None: """Write a run's changed files into this checkout and its logs to logs_dir.""" diff --git a/experiments/receipts/1073-scf-education-debt-fit.json b/experiments/receipts/1073-scf-education-debt-fit.json index 5a17f1b568..2297b236c2 100644 --- a/experiments/receipts/1073-scf-education-debt-fit.json +++ b/experiments/receipts/1073-scf-education-debt-fit.json @@ -1,7 +1,11 @@ { "schema_version": 1, "issue": "PolicyEngine/microcosm#1073", - "generated_at": "2026-10-12", + "generated_at": "2026-10-21", + "provenance": { + "commit": "e35c4b0f911883585c150d2d6417dc01da2bb9d9", + "run": "20261021T133110Z-e35c4b0f9" + }, "scf_files": { "rscfp2025.dta": { "url": "https://www.federalreserve.gov/econres/files/scfp2025s.zip", @@ -39,7 +43,7 @@ "-7 other": 0.0 }, "whose_education_codes_outside_the_rule": [], - "federal_education_debt_after_x7179_rule": 1005887392198.396, + "federal_education_debt_after_x7179_rule": 1005887392198.3959, "remaining_loans_without_detail_dollars": 0.0 }, "scf_2025_observed": { @@ -49,8 +53,8 @@ "mean_among_borrowers": 41052.27209260245, "median_among_borrowers": 25000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 1005887392198.396, - "parent_plus_share_of_total": 0.13750075307389342 + "federal_education_debt_total": 1005887392198.3959, + "parent_plus_share_of_total": 0.13750075307389345 }, "household_balance_bands": { "under_5k": 0.09639294713632403, @@ -175,7 +179,7 @@ "50_to_61": 0.1615926964748789, "62_and_older": 0.06902425352362417 }, - "graduate_share_of_own_original_principal": 0.47738241507384116, + "graduate_share_of_own_original_principal": 0.4773824150738412, "ibr_new_borrower_share": 0.5915143468879298 } }, @@ -183,7 +187,7 @@ "folds": 5, "seed": 0, "n_estimators": 100, - "production_variant": "shared_8_plus_household_income", + "production_variant": "build_without_interest_and_ss_pension", "variants": { "shared_8_without_age": { "predictors": [ @@ -328,7 +332,7 @@ "50_to_61": 0.21093410606503074, "62_and_older": 0.11268289035889188 }, - "graduate_share_of_own_original_principal": 0.4521386518636319, + "graduate_share_of_own_original_principal": 0.45213865186363184, "ibr_new_borrower_share": 0.5904131759381358 } }, @@ -364,7 +368,7 @@ "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, "federal_education_debt_total": 936446699439.1069, - "parent_plus_share_of_total": 0.12480448512425421 + "parent_plus_share_of_total": 0.12480448512425425 }, "household_balance_bands": { "under_5k": 0.09343877143258432, @@ -526,7 +530,7 @@ "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, "federal_education_debt_total": 915853474784.7556, - "parent_plus_share_of_total": 0.11558498718060217 + "parent_plus_share_of_total": 0.11558498718060221 }, "household_balance_bands": { "under_5k": 0.08974701999643789, @@ -814,7 +818,7 @@ "50_to_61": 0.1615925449287847, "62_and_older": 0.06370079077949123 }, - "graduate_share_of_own_original_principal": 0.45646219259032567, + "graduate_share_of_own_original_principal": 0.4564621925903257, "ibr_new_borrower_share": 0.5822440301014113 } }, @@ -830,6 +834,831 @@ "borrower_age_band_tvd": 0.023231363094113566, "household_rank_correlation": 0.20794745847766363 } + }, + "build_without_interest_and_ss_pension": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.1785134113597766, + "mean_among_borrowers": 38129.7481875102, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 918653496595.8906, + "parent_plus_share_of_total": 0.12228737578012734 + }, + "household_balance_bands": { + "under_5k": 0.09079966274592437, + "5k_to_10k": 0.1560993205378036, + "10k_to_20k": 0.17784594975720502, + "20k_to_40k": 0.24170300969285868, + "40k_to_60k": 0.12229052439445375, + "60k_to_80k": 0.0975892083544051, + "80k_to_100k": 0.045649331362645214, + "100k_to_200k": 0.04868017050483829, + "200k_plus": 0.019342822649865955 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.24959547861591452, + "mean_among_borrowers": 30234.050260438173, + "median_among_borrowers": 14000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.36900446914360446, + "mean_among_borrowers": 35840.91733887638, + "median_among_borrowers": 23000.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2743786349921627, + "mean_among_borrowers": 38624.21584848466, + "median_among_borrowers": 22900.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.14829262594950599, + "mean_among_borrowers": 40029.75297693061, + "median_among_borrowers": 27000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03498656926046411, + "mean_among_borrowers": 48212.80171837554, + "median_among_borrowers": 26997.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.1672189930591373, + "mean_among_borrowers": 36898.38867717323, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.2324868349178142, + "mean_among_borrowers": 44763.01942694598, + "median_among_borrowers": 25000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.16156695468894305, + "mean_among_borrowers": 35460.090234059295, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.18853789989668854, + "mean_among_borrowers": 36660.59407673795, + "median_among_borrowers": 21000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.10379714978765006, + "mean_among_borrowers": 34624.24650800998, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.17803919929996376, + "mean_among_borrowers": 37509.212621969265, + "median_among_borrowers": 21000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17956752806930856, + "mean_among_borrowers": 35860.25899631823, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.2632482800176951, + "mean_among_borrowers": 38126.0224509717, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16793147929877925, + "mean_among_borrowers": 43386.556076636596, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 25709142.2609221, + "mean_balance": 35732.56109723443, + "median_balance": 21000.0, + "balance_bands": { + "under_5k": 0.10244159338676109, + "5k_to_10k": 0.16637242925127055, + "10k_to_20k": 0.18690627049147995, + "20k_to_40k": 0.23582143486648904, + "40k_to_60k": 0.11958046299653502, + "60k_to_80k": 0.08637580964972973, + "80k_to_100k": 0.039420314567368823, + "100k_to_200k": 0.046956700106719984, + "200k_plus": 0.01612498468364574 + }, + "age_bands": { + "24_and_younger": 0.0646454470512797, + "25_to_34": 0.3387609953816956, + "35_to_49": 0.37960479585619167, + "50_to_61": 0.15141879980849401, + "62_and_older": 0.06556996190233892 + }, + "graduate_share_of_own_original_principal": 0.45012054963197445, + "ibr_new_borrower_share": 0.5931987024130821 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.30361567622269603, + "mean_among_borrowers_error_pct": -7.119030826113226, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03158207267342461, + "head_age_band_share_mae_pp": 1.2596405967540016, + "race_share_mae_pp": 1.069077118976515, + "income_quintile_share_mae_pp": 0.6750791601254419, + "borrower_balance_band_tvd": 0.032948871945896414, + "borrower_age_band_tvd": 0.014803476410333236, + "household_rank_correlation": 0.16787602335617488 + } + }, + "build_plus_education": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income", + "educ" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17234700775018835, + "mean_among_borrowers": 38142.78924085034, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 887223715061.3356, + "parent_plus_share_of_total": 0.11798681584091122 + }, + "household_balance_bands": { + "under_5k": 0.094500931393709, + "5k_to_10k": 0.1398997895474437, + "10k_to_20k": 0.19368288257466282, + "20k_to_40k": 0.23884340819629526, + "40k_to_60k": 0.1352262887879038, + "60k_to_80k": 0.09436055969371929, + "80k_to_100k": 0.03446366840523407, + "100k_to_200k": 0.0523437592826446, + "200k_plus": 0.0166787121183875 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.24556070748817072, + "mean_among_borrowers": 29711.57837171095, + "median_among_borrowers": 13345.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.37451305639803345, + "mean_among_borrowers": 36295.97859113742, + "median_among_borrowers": 23900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.25660807838739297, + "mean_among_borrowers": 38679.04268701929, + "median_among_borrowers": 21600.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.13992324697554848, + "mean_among_borrowers": 38624.20371347948, + "median_among_borrowers": 25000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03311810732872758, + "mean_among_borrowers": 50709.27699844198, + "median_among_borrowers": 30000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.16168801258725676, + "mean_among_borrowers": 37236.74657199004, + "median_among_borrowers": 22900.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.20914133036844892, + "mean_among_borrowers": 44501.95583092358, + "median_among_borrowers": 26000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.16318201246763214, + "mean_among_borrowers": 31794.600738997113, + "median_among_borrowers": 21000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.19023526055566922, + "mean_among_borrowers": 39178.989837148605, + "median_among_borrowers": 20000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.10129901303895304, + "mean_among_borrowers": 42632.5471947818, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.17021674724220964, + "mean_among_borrowers": 39357.84941524378, + "median_among_borrowers": 20100.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17533106736677012, + "mean_among_borrowers": 31912.544492445948, + "median_among_borrowers": 20000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.2535480923849969, + "mean_among_borrowers": 35780.224907810196, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16135595041628936, + "mean_among_borrowers": 44524.142074980824, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 24672223.015597936, + "mean_balance": 35960.42863670725, + "median_balance": 20333.0, + "balance_bands": { + "under_5k": 0.1094406791705482, + "5k_to_10k": 0.15004936475183045, + "10k_to_20k": 0.19555413431333177, + "20k_to_40k": 0.23372872974097667, + "40k_to_60k": 0.12914925746852132, + "60k_to_80k": 0.08602472899703802, + "80k_to_100k": 0.031224250567329498, + "100k_to_200k": 0.051343490636136885, + "200k_plus": 0.013485364354287193 + }, + "age_bands": { + "24_and_younger": 0.06286336983939379, + "25_to_34": 0.3564829429265149, + "35_to_49": 0.36706810417884167, + "50_to_61": 0.14875323302634258, + "62_and_older": 0.06483235002890703 + }, + "graduate_share_of_own_original_principal": 0.45388442861260153, + "ibr_new_borrower_share": 0.5880863294842776 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.9202560371815216, + "mean_among_borrowers_error_pct": -7.087263879546368, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.02820605085678676, + "head_age_band_share_mae_pp": 1.7487117716451235, + "race_share_mae_pp": 1.7889316725003699, + "income_quintile_share_mae_pp": 0.920292137753204, + "borrower_balance_band_tvd": 0.029131316884503575, + "borrower_age_band_tvd": 0.030743346743266606, + "household_rank_correlation": 0.20717704021713643 + } + } + } + }, + "host_settings": { + "first_receipt": { + "label": "First receipt", + "definition": { + "reference_person": "oldest_adult", + "partnered": "married_spouse", + "children": "of_reference", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14412999386354491, + "mean_among_borrowers": 44483.09961323457, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 785628046119.6727, + "parent_plus_share_of_total": 0.21495679635292875 + }, + "household_balance_bands": { + "under_5k": 0.09597901426395242, + "5k_to_10k": 0.1620676904114682, + "10k_to_20k": 0.15178493618645325, + "20k_to_40k": 0.22796648343692355, + "40k_to_60k": 0.09915026443778263, + "60k_to_80k": 0.10849192930212849, + "80k_to_100k": 0.056043492646084624, + "100k_to_200k": 0.06743544596300373, + "200k_plus": 0.031080743352203163 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.2267174320421409, + "mean_among_borrowers": 18670.63953159211, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2669429791293293, + "mean_among_borrowers": 36447.86184035785, + "median_among_borrowers": 20200.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22339688829964577, + "mean_among_borrowers": 44315.447461546944, + "median_among_borrowers": 25670.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14218578094851128, + "mean_among_borrowers": 49552.354866035355, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03655929815086838, + "mean_among_borrowers": 65818.86253190483, + "median_among_borrowers": 38000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13333034770117472, + "mean_among_borrowers": 42577.59434933696, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902667788102, + "mean_among_borrowers": 47993.699863516216, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12291729609744638, + "mean_among_borrowers": 46285.94561484015, + "median_among_borrowers": 24764.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15337638725400463, + "mean_among_borrowers": 47893.571558628166, + "median_among_borrowers": 24000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12721563759800417, + "mean_among_borrowers": 38171.39283093021, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1358771691149805, + "mean_among_borrowers": 45461.58729969715, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13382646407799811, + "mean_among_borrowers": 40922.60303950533, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19993030155269514, + "mean_among_borrowers": 45004.488576349744, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12378308052696536, + "mean_among_borrowers": 52902.03665110157, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18461703.3923496, + "mean_balance": 42554.4723270352, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.1039144177980828, + "5k_to_10k": 0.16665917220324747, + "10k_to_20k": 0.15515901493993045, + "20k_to_40k": 0.23029463650667883, + "40k_to_60k": 0.10426754245310213, + "60k_to_80k": 0.09578243033185131, + "80k_to_100k": 0.05074417014558102, + "100k_to_200k": 0.06376833884555547, + "200k_plus": 0.029410276775970686 + }, + "age_bands": { + "24_and_younger": 0.03769253987750687, + "25_to_34": 0.2662405413523508, + "35_to_49": 0.39817213354338704, + "50_to_61": 0.21071813444930676, + "62_and_older": 0.08717665077744856 + }, + "graduate_share_of_own_original_principal": 0.5360724443816975, + "ibr_new_borrower_share": 0.605734686749079 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7419574258458654, + "mean_among_borrowers_error_pct": 8.357217142313411, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.061856909978440955, + "head_age_band_share_mae_pp": 3.5543373606609885, + "race_share_mae_pp": 2.8343426729118955, + "income_quintile_share_mae_pp": 4.2522526924143715, + "borrower_balance_band_tvd": 0.06971797938468582, + "borrower_age_band_tvd": 0.08466988479278444 + } + }, + "scf_codings_all_candidates": { + "label": "SCF codings, all predictors", + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14453729401731658, + "mean_among_borrowers": 42173.85469504409, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 746948718213.9073, + "parent_plus_share_of_total": 0.15519605817424037 + }, + "household_balance_bands": { + "under_5k": 0.10775015906187184, + "5k_to_10k": 0.1528994221663368, + "10k_to_20k": 0.15878382717932804, + "20k_to_40k": 0.23707174823778418, + "40k_to_60k": 0.11254827709408233, + "60k_to_80k": 0.09506114425698398, + "80k_to_100k": 0.044996423198641146, + "100k_to_200k": 0.05939892961997767, + "200k_plus": 0.03149006918499395 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.2109485663269708, + "mean_among_borrowers": 22164.502413760794, + "median_among_borrowers": 14000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.2709152074679265, + "mean_among_borrowers": 36647.47528338518, + "median_among_borrowers": 21600.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.20910334189997756, + "mean_among_borrowers": 42586.38683331958, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.1331061797690564, + "mean_among_borrowers": 49336.50631455528, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.03120975131828736, + "mean_among_borrowers": 58478.07907213402, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.13312784049029713, + "mean_among_borrowers": 40718.755628997955, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.22788477777157826, + "mean_among_borrowers": 47824.23980122162, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11755065774394279, + "mean_among_borrowers": 39599.36261332915, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.1698883489488461, + "mean_among_borrowers": 44027.69290376679, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.13612388616760965, + "mean_among_borrowers": 38386.48367756073, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.1459793402381904, + "mean_among_borrowers": 42114.80663102074, + "median_among_borrowers": 23000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.13033686972701333, + "mean_among_borrowers": 39083.07983863153, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.1919836615881704, + "mean_among_borrowers": 42068.527873712716, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.11826299055685831, + "mean_among_borrowers": 50185.22911179778, + "median_among_borrowers": 26814.0 + } + }, + "borrowers": { + "borrower_weight": 18531371.849430196, + "mean_balance": 40307.25432973677, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.11848284445361304, + "5k_to_10k": 0.15391028318710992, + "10k_to_20k": 0.16124932842055162, + "20k_to_40k": 0.23782638366100617, + "40k_to_60k": 0.11453284638670623, + "60k_to_80k": 0.09065262966356538, + "80k_to_100k": 0.0390975463353386, + "100k_to_200k": 0.05475398408033307, + "200k_plus": 0.029494153811776037 + }, + "age_bands": { + "24_and_younger": 0.05515156793888616, + "25_to_34": 0.3089032338721271, + "35_to_49": 0.3729683379284752, + "50_to_61": 0.19389979448680775, + "62_and_older": 0.06907706577370382 + }, + "graduate_share_of_own_original_principal": 0.4975010554094781, + "ibr_new_borrower_share": 0.6165370510095016 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7012274104686993, + "mean_among_borrowers_error_pct": 2.7320841095266557, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.040156439181474324, + "head_age_band_share_mae_pp": 4.169585550417035, + "race_share_mae_pp": 2.569878492307957, + "income_quintile_share_mae_pp": 4.567500727054615, + "borrower_balance_band_tvd": 0.04019409665288553, + "borrower_age_band_tvd": 0.03235991026200852 + } + }, + "build": { + "label": "Build", + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16413671000206656, + "mean_among_borrowers": 42680.26879514089, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 858421208954.9309, + "parent_plus_share_of_total": 0.16758194869071996 + }, + "household_balance_bands": { + "under_5k": 0.1179407532730876, + "5k_to_10k": 0.1324473758996882, + "10k_to_20k": 0.1827494270561508, + "20k_to_40k": 0.23417620772637385, + "40k_to_60k": 0.10433535383061796, + "60k_to_80k": 0.0976346772652531, + "80k_to_100k": 0.0379014108379451, + "100k_to_200k": 0.05781213394341271, + "200k_plus": 0.03500266016747067 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.21797195752357604, + "mean_among_borrowers": 20475.641978208863, + "median_among_borrowers": 12662.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28107974022687005, + "mean_among_borrowers": 34573.95711939027, + "median_among_borrowers": 20100.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.24429588401566177, + "mean_among_borrowers": 44478.05328490035, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16352578267462928, + "mean_among_borrowers": 48394.24067058906, + "median_among_borrowers": 28742.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.037347739302272136, + "mean_among_borrowers": 61031.6733341336, + "median_among_borrowers": 34000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15291016385941802, + "mean_among_borrowers": 42266.19101485241, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.24861685783459786, + "mean_among_borrowers": 45500.64480289935, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.13835157436352993, + "mean_among_borrowers": 37031.934121143146, + "median_among_borrowers": 19200.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.18407544859054642, + "mean_among_borrowers": 48161.13603160578, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14257834053880739, + "mean_among_borrowers": 43328.84042017116, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.15791199105287693, + "mean_among_borrowers": 39522.35712436794, + "median_among_borrowers": 20000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14719109419650156, + "mean_among_borrowers": 38670.33311547162, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.23072142160681058, + "mean_among_borrowers": 41126.05793612391, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1422795004455311, + "mean_among_borrowers": 52202.66543686702, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 21223403.433088087, + "mean_balance": 40446.915673129966, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.12533388559616318, + "5k_to_10k": 0.13601201669515878, + "10k_to_20k": 0.18997031373843593, + "20k_to_40k": 0.2351308957545589, + "40k_to_60k": 0.10824433293789903, + "60k_to_80k": 0.08668533742450127, + "80k_to_100k": 0.03163070469899026, + "100k_to_200k": 0.05604778033785523, + "200k_plus": 0.03094473281643728 + }, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257 + }, + "graduate_share_of_own_original_principal": 0.48269344604304165, + "ibr_new_borrower_share": 0.6147006160485154 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.7412858119937007, + "mean_among_borrowers_error_pct": 3.965667719599386, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.046105502753231414, + "head_age_band_share_mae_pp": 2.3908244112012222, + "race_share_mae_pp": 0.9415277758683394, + "income_quintile_share_mae_pp": 2.865766910648814, + "borrower_balance_band_tvd": 0.04473225112436991, + "borrower_age_band_tvd": 0.049566199767822304 } } }, @@ -874,150 +1703,150 @@ "fit": { "households": { "household_weight": 122537091.8856833, - "share_with_federal_education_debt": 0.14412999386354491, - "mean_among_borrowers": 44483.09961323457, - "median_among_borrowers": 25000.0, + "share_with_federal_education_debt": 0.16413671000206656, + "mean_among_borrowers": 42680.26879514089, + "median_among_borrowers": 23000.0, "total_household_weight": 122537091.8856833, - "federal_education_debt_total": 785628046119.6729, - "parent_plus_share_of_total": 0.21495679635292878 + "federal_education_debt_total": 858421208954.9309, + "parent_plus_share_of_total": 0.16758194869071996 }, "household_balance_bands": { - "under_5k": 0.09597901426395242, - "5k_to_10k": 0.1620676904114682, - "10k_to_20k": 0.15178493618645325, - "20k_to_40k": 0.22796648343692355, - "40k_to_60k": 0.09915026443778263, - "60k_to_80k": 0.10849192930212849, - "80k_to_100k": 0.056043492646084624, - "100k_to_200k": 0.06743544596300373, - "200k_plus": 0.031080743352203163 + "under_5k": 0.1179407532730876, + "5k_to_10k": 0.1324473758996882, + "10k_to_20k": 0.1827494270561508, + "20k_to_40k": 0.23417620772637385, + "40k_to_60k": 0.10433535383061796, + "60k_to_80k": 0.0976346772652531, + "80k_to_100k": 0.0379014108379451, + "100k_to_200k": 0.05781213394341271, + "200k_plus": 0.03500266016747067 }, "by_head_age_band": { "24_and_younger": { - "household_weight": 2628112.658323748, - "share_with_federal_education_debt": 0.2267174320421409, - "mean_among_borrowers": 18670.63953159211, - "median_among_borrowers": 6000.0 + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.21797195752357604, + "mean_among_borrowers": 20475.641978208863, + "median_among_borrowers": 12662.0 }, "25_to_34": { - "household_weight": 17516284.279591538, - "share_with_federal_education_debt": 0.2669429791293293, - "mean_among_borrowers": 36447.86184035785, - "median_among_borrowers": 20200.0 + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28107974022687005, + "mean_among_borrowers": 34573.95711939027, + "median_among_borrowers": 20100.0 }, "35_to_49": { - "household_weight": 30963807.6460925, - "share_with_federal_education_debt": 0.22339688829964577, - "mean_among_borrowers": 44315.447461546944, - "median_among_borrowers": 25670.0 + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.24429588401566177, + "mean_among_borrowers": 44478.05328490035, + "median_among_borrowers": 23000.0 }, "50_to_61": { - "household_weight": 27085763.89545781, - "share_with_federal_education_debt": 0.14218578094851128, - "mean_among_borrowers": 49552.354866035355, - "median_among_borrowers": 29000.0 + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16352578267462928, + "mean_among_borrowers": 48394.24067058906, + "median_among_borrowers": 28742.0 }, "62_and_older": { - "household_weight": 44343123.406217694, - "share_with_federal_education_debt": 0.03655929815086838, - "mean_among_borrowers": 65818.86253190483, - "median_among_borrowers": 38000.0 + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.037347739302272136, + "mean_among_borrowers": 61031.6733341336, + "median_among_borrowers": 34000.0 } }, "by_race": { "white_non_hispanic": { - "household_weight": 80232895.31111473, - "share_with_federal_education_debt": 0.13333034770117472, - "mean_among_borrowers": 42577.59434933696, - "median_among_borrowers": 25000.0 + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15291016385941802, + "mean_among_borrowers": 42266.19101485241, + "median_among_borrowers": 23000.0 }, "black_non_hispanic": { - "household_weight": 13933034.37446032, - "share_with_federal_education_debt": 0.22824902667788102, - "mean_among_borrowers": 47993.699863516216, + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.24861685783459786, + "mean_among_borrowers": 45500.64480289935, "median_among_borrowers": 28000.0 }, "hispanic": { - "household_weight": 18643938.505382426, - "share_with_federal_education_debt": 0.12291729609744638, - "mean_among_borrowers": 46285.94561484015, - "median_among_borrowers": 24764.0 + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.13835157436352993, + "mean_among_borrowers": 37031.934121143146, + "median_among_borrowers": 19200.0 }, "other_or_multiple": { - "household_weight": 9727223.69472581, - "share_with_federal_education_debt": 0.15337638725400463, - "mean_among_borrowers": 47893.571558628166, - "median_among_borrowers": 24000.0 + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.18407544859054642, + "mean_among_borrowers": 48161.13603160578, + "median_among_borrowers": 18000.0 } }, "by_income_quintile": { "1": { - "household_weight": 24506450.631004497, - "share_with_federal_education_debt": 0.12721563759800417, - "mean_among_borrowers": 38171.39283093021, + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14257834053880739, + "mean_among_borrowers": 43328.84042017116, "median_among_borrowers": 20000.0 }, "2": { - "household_weight": 24508606.10032361, - "share_with_federal_education_debt": 0.1358771691149805, - "mean_among_borrowers": 45461.58729969715, - "median_among_borrowers": 26000.0 + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.15791199105287693, + "mean_among_borrowers": 39522.35712436794, + "median_among_borrowers": 20000.0 }, "3": { - "household_weight": 24502185.328004334, - "share_with_federal_education_debt": 0.13382646407799811, - "mean_among_borrowers": 40922.60303950533, + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14719109419650156, + "mean_among_borrowers": 38670.33311547162, "median_among_borrowers": 23000.0 }, "4": { - "household_weight": 24513536.74023802, - "share_with_federal_education_debt": 0.19993030155269514, - "mean_among_borrowers": 45004.488576349744, - "median_among_borrowers": 25000.0 + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.23072142160681058, + "mean_among_borrowers": 41126.05793612391, + "median_among_borrowers": 26000.0 }, "5": { - "household_weight": 24506313.086112835, - "share_with_federal_education_debt": 0.12378308052696536, - "mean_among_borrowers": 52902.03665110157, - "median_among_borrowers": 28000.0 + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1422795004455311, + "mean_among_borrowers": 52202.66543686702, + "median_among_borrowers": 23000.0 } }, "borrowers": { - "borrower_weight": 18461703.3923496, - "mean_balance": 42554.4723270352, - "median_balance": 24000.0, + "borrower_weight": 21223403.433088087, + "mean_balance": 40446.915673129966, + "median_balance": 22000.0, "balance_bands": { - "under_5k": 0.1039144177980828, - "5k_to_10k": 0.16665917220324747, - "10k_to_20k": 0.15515901493993045, - "20k_to_40k": 0.23029463650667883, - "40k_to_60k": 0.10426754245310213, - "60k_to_80k": 0.09578243033185131, - "80k_to_100k": 0.05074417014558102, - "100k_to_200k": 0.06376833884555547, - "200k_plus": 0.029410276775970686 + "under_5k": 0.12533388559616318, + "5k_to_10k": 0.13601201669515878, + "10k_to_20k": 0.18997031373843593, + "20k_to_40k": 0.2351308957545589, + "40k_to_60k": 0.10824433293789903, + "60k_to_80k": 0.08668533742450127, + "80k_to_100k": 0.03163070469899026, + "100k_to_200k": 0.05604778033785523, + "200k_plus": 0.03094473281643728 }, "age_bands": { - "24_and_younger": 0.03769253987750687, - "25_to_34": 0.2662405413523508, - "35_to_49": 0.39817213354338704, - "50_to_61": 0.21071813444930676, - "62_and_older": 0.08717665077744856 - }, - "graduate_share_of_own_original_principal": 0.5360724443816975, - "ibr_new_borrower_share": 0.605734686749079 + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257 + }, + "graduate_share_of_own_original_principal": 0.48269344604304165, + "ibr_new_borrower_share": 0.6147006160485154 } }, "metrics_vs_scf": { - "share_with_debt_error_pp": -3.7419574258458654, - "mean_among_borrowers_error_pct": 8.357217142313411, - "median_among_borrowers_error_pct": 0.0, - "household_balance_band_tvd": 0.061856909978440955, - "head_age_band_share_mae_pp": 3.5543373606609885, - "race_share_mae_pp": 2.8343426729118955, - "income_quintile_share_mae_pp": 4.2522526924143715, - "borrower_balance_band_tvd": 0.06971797938468582, - "borrower_age_band_tvd": 0.08466988479278444 + "share_with_debt_error_pp": -1.7412858119937007, + "mean_among_borrowers_error_pct": 3.965667719599386, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.046105502753231414, + "head_age_band_share_mae_pp": 2.3908244112012222, + "race_share_mae_pp": 0.9415277758683394, + "income_quintile_share_mae_pp": 2.865766910648814, + "borrower_balance_band_tvd": 0.04473225112436991, + "borrower_age_band_tvd": 0.049566199767822304 } } } diff --git a/test_support/microcosm_data/stored_inputs.py b/test_support/microcosm_data/stored_inputs.py index 4206bdf943..d42aed8769 100644 --- a/test_support/microcosm_data/stored_inputs.py +++ b/test_support/microcosm_data/stored_inputs.py @@ -155,9 +155,20 @@ def _column_strategies(st): #: Register entries that postdate the files examined for #1026 and rest on a #: live producer instead: the parent ids (microcosm#884), bound to -#: eligibility_inputs.US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS by +#: eligibility_inputs.US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS, and the four +#: federal student loan inputs (microcosm#1073), bound to +#: scf_education_debt.US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, both by #: test_us_stored_input_register.py (microcosm-data cannot import the build). -_PRODUCER_BOUND_ENTRIES = frozenset({"parent_1_id", "parent_2_id"}) +_PRODUCER_BOUND_ENTRIES = frozenset( + { + "parent_1_id", + "parent_2_id", + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share", + } +) #: The role columns policyengine-core's ``build_from_dataset`` reads to build From fcbd704c964bcbfd1292b4ab4ccb702ea8415e72 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Wed, 21 Oct 2026 11:05:01 -0400 Subject: [PATCH 08/17] Re-derive the spec-engine identity values the education-debt stage moves (#1073) The scf_wealth education-debt declaration adds 145 configuration fields to the US bundle (141 authored, 4 resolved). Modal probe run 20261021T145336Z-b1a461f19 printed every value below from the code; none is derived by hand. It also diffed the field universe against origin/main: all 159 added and 14 removed pointers belong to exactly one claim, and no field is unclaimed or claimed twice. - field_usage: 42,329 configuration / 32,545 authored / 9,784 resolved (was 42,184 / 32,404 / 9,780); claim pins source_stages 1,845 (the two 2025 artifacts and six operations of scf_wealth), vintage_records 120 and resolved_vintage_authorities 67 (scf_2025), calibration 316 (the four person outputs in the QRF tail-concentration list), each with its observed digest; ledger mode and generation-0 effect counts in the field-usage and coverage-tool tests. - inventory_coverage: source_manifest, late_resource_semantics, tail and full_checkpoint digests; tail control fields 938 and resolved references 336. - The pool tool's bundle spec_sha256 (a6f254a8...). - experiments/modal_1073.py: the spec step first runs the bundle generator and the coverage attestation in --check mode. The coverage tool's pointer-inventory digest and the regenerated docs/evidence/spec-engine/us-f0-coverage.json follow in the next commit. Co-Authored-By: Claude Opus 5.5 (1M context) --- experiments/modal_1073.py | 9 +++++--- .../build/spec_engine/field_usage.py | 22 +++++++++---------- .../build/spec_engine/inventory_coverage.py | 16 +++++++------- .../us/test_spec_engine_coverage_tool.py | 18 +++++++-------- .../shared/test_spec_engine_field_usage.py | 16 +++++++------- .../us/test_us_multispine_pool_tool.py | 2 +- 6 files changed, 43 insertions(+), 40 deletions(-) diff --git a/experiments/modal_1073.py b/experiments/modal_1073.py index c4a4b575b6..8c636912f6 100644 --- a/experiments/modal_1073.py +++ b/experiments/modal_1073.py @@ -23,9 +23,10 @@ ``tools/generate_us_bundle_from_constants.py`` after a hand edit of ``source_stages.json``; - ``tests``: the stage's unit tests; -- ``spec``: the engine-free and engine-contract tests that read the spec - bundle, the stored-input register and the SCF stages, plus the release - tool's education-debt tests; +- ``spec``: the bundle generator and the spec-engine coverage attestation in + ``--check`` mode, then the engine-free and engine-contract tests that read + the spec bundle, the stored-input register and the SCF stages, plus the + release tool's education-debt tests; - ``hostfit``, ``shift``, ``explore``: the #1073 diagnostics; - ``receipt``: the fit receipt, labelled with the run id. @@ -93,6 +94,8 @@ ] ], "spec": [ + ["{python}", "tools/generate_us_bundle_from_constants.py", "--check"], + ["{python}", "tools/spec_engine_coverage.py", "--check"], [ "{python}", "-m", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 56f39195cf..db52e73774 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_404 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_184 +EXPECTED_AUTHORED_FIELD_COUNT = 32_545 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_784 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_329 class FieldUsageError(AssertionError): @@ -361,8 +361,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "36564747627fb0058bb99e8e8c4397e41d8ecc472a20c49d4b6d7df1003a80ec", ), "calibration": ( - 312, - "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + 316, + "0212e52e2bf1e5996fe6b4785b2d74f2fb5680ec38b258c6ad4dbeadbd181a1b", ), "catalog_columns": ( 1_673, @@ -461,8 +461,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", ), "resolved_vintage_authorities": ( - 63, - "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", + 67, + "3f49902e745368ce8df0ad338a1697b76fd3299d35959977d36a8c09690ffb10", ), "selection": ( 87, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_716, - "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + 1_845, + "4142b1037cddd9621b70eb9677be89510474475d9a705f1cedf7a32d7bf64db8", ), "spine_assembly_household_mass_shares": ( 2, @@ -533,8 +533,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d53096196db4c34da260cce2f35af8e7ba67f978448656c602ee5e17529dc4e0", ), "vintage_records": ( - 112, - "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", + 120, + "825b4ab09602353fc4d6c24b6f7654f7582502c7acf963b7ff6c117b07208eb4", ), } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 2a38471603..92621a2678 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", + "late_resource_semantics": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -363,9 +363,9 @@ # RNG rules and the ordered QRF targets are unchanged. "seed_map": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", "seed_protocol": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", - "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "source_manifest": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", - "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", + "tail": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", } INVENTORY_REPORT_SCHEMA_VERSION = 1 @@ -435,7 +435,7 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -445,7 +445,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -1413,7 +1413,7 @@ def add( add( "capital_gains_tail_contract_exact", clauses={ - "tail terminal field count differs": tail_count == 934, + "tail terminal field count differs": tail_count == 938, "resolved calibration tails differ": _json_equal( legacy.get("calibration_tail_contracts"), expected_legacy["calibration_tail_contracts"], @@ -1441,7 +1441,7 @@ def add( "legacy_adapter.stacked_checkpoint_static_components", ), observed={"terminal_fields": tail_count, "sha256": sha256_json(tail_bundle)}, - expected={"terminal_fields": 934, "sha256": EXPECTED_HASHES["tail"]}, + expected={"terminal_fields": 938, "sha256": EXPECTED_HASHES["tail"]}, ) add( diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 7be8062b02..5e50ed272e 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_184 - assert fields["authored_normative_field_count"] == 32_404 - assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_184 + assert fields["configuration_field_count"] == 42_329 + assert fields["authored_normative_field_count"] == 32_545 + assert fields["resolved_binding_field_count"] == 9_784 + assert fields["consumed_field_count"] == 42_329 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_143, + "compiler_semantic": 27_729, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_631, + "no_generation0_effect": 3_698, } inventory = coverage_report["inventory_coverage"] @@ -107,7 +107,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["missing_items"] == [] assert inventory["counts"]["producer_inputs"] == 2_744 assert inventory["counts"]["ownership_rows"] == 18 - assert inventory["counts"]["tail_control_fields"] == 934 + assert inventory["counts"]["tail_control_fields"] == 938 assert inventory["counts"]["seed_owner_bindings"] == 112 diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index c5ec18bb51..f87e5a9571 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_184 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_329 assert field_ledger.source_counts == { - "authored": 32_404, - "resolved_bindings": 9_780, + "authored": 32_545, + "resolved_bindings": 9_784, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_143, + "compiler_semantic": 27_729, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_631, + "no_generation0_effect": 3_698, } - assert len({field.pointer for field in field_ledger.fields}) == 42_184 + assert len({field.pointer for field in field_ledger.fields}) == 42_329 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 5b1861afdd..157934c408 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f", + "spec_sha256": "a6f254a8f98da41e2df3f2c54f6488759f84f95c42251064020ab597861751c0", }, } From 05856e6ca8b681bb3604a373394e07a0625cf9fe Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Wed, 21 Oct 2026 11:14:37 -0400 Subject: [PATCH 09/17] Pin the coverage pointer inventory and regenerate the F0 attestation (#1073) Modal probe run 20261021T150509Z-fcbd704c9, with the previous commit's field-usage and inventory pins in place, printed the coverage tool's pointer-inventory digest (37ce78d6...) and regenerated docs/evidence/spec-engine/us-f0-coverage.json with tools/spec_engine_coverage.py; --check then reported 42329/42329 configuration fields and 41/41 inventory checks. The bundle spec_sha256 stays a6f254a8.... Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/evidence/spec-engine/us-f0-coverage.json | 66 +++++++++---------- tools/spec_engine_coverage.py | 2 +- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 978c19dcd8..d7ba658b05 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "bd20f447db53159f719eba20c13a44efb825dc40fcc1db1d4f661ebbd6a7916b", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32545, "claim_count": 49, "claims": [ { @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 316, + "pointer_sha256": "0212e52e2bf1e5996fe6b4785b2d74f2fb5680ec38b258c6ad4dbeadbd181a1b", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1716, - "pointer_sha256": "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + "pointer_count": 1845, + "pointer_sha256": "4142b1037cddd9621b70eb9677be89510474475d9a705f1cedf7a32d7bf64db8", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -705,8 +705,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 112, - "pointer_sha256": "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", + "pointer_count": 120, + "pointer_sha256": "825b4ab09602353fc4d6c24b6f7654f7582502c7acf963b7ff6c117b07208eb4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1vintages.yaml", @@ -764,29 +764,29 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 63, - "pointer_sha256": "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", + "pointer_count": 67, + "pointer_sha256": "3f49902e745368ce8df0ad338a1697b76fd3299d35959977d36a8c09690ffb10", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/vintage_authorities", "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42329, + "consumed_field_count": 42329, "generation0_effect_counts": { - "legacy_behavior": 38498, - "no_generation0_effect": 3686 + "legacy_behavior": 38631, + "no_generation0_effect": 3698 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27729, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14010 + "legacy_behavior": 14143 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", - "resolved_binding_field_count": 9780, + "pointer_inventory_sha256": "37ce78d62b0eb4761eec78a22da637fc242c8adc516bb69ab6154430051c33e1", + "resolved_binding_field_count": 9784, "unused_field_count": 0 }, "inventory_coverage": { @@ -814,7 +814,7 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -824,7 +824,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", + "terminal_fields": 938 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", + "terminal_fields": 938 }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", "stages": 37 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "6fcc5e3cd9b9a4063ba211cfc499321b0e84807a60e28f76492b4e16a8a15678" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f" + "spec_sha256": "a6f254a8f98da41e2df3f2c54f6488759f84f95c42251064020ab597861751c0" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f" + "spec_sha256": "a6f254a8f98da41e2df3f2c54f6488759f84f95c42251064020ab597861751c0" }, "status": "pass" } diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index c066d573ce..ed3539039c 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7" + "37ce78d62b0eb4761eec78a22da637fc242c8adc516bb69ab6154430051c33e1" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 83a42f85d2b5bda3a8f1b0e1bb01e942e82bc5ea Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Wed, 21 Oct 2026 12:48:47 -0400 Subject: [PATCH 10/17] Re-pin the inventory test's resolved-reference and tail-control counts (#1073) test_support/microcosm_build/spec_engine_inventory_coverage.py keeps its own copy of the reviewed inventory counts. Modal run 20261021T151444Z-05856e6ca reported the two the stage moves, matching inventory_coverage.py and probe run 20261021T145336Z-b1a461f19: resolved references 336 (the scf_2025 vintage) and tail control fields 938 (the four outputs in the QRF tail-concentration list). Everything else in that run passed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../microcosm_build/spec_engine_inventory_coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 1d051f5c4d..0b20fdbc36 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -123,7 +123,7 @@ def binding() -> dict[str, object]: "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -133,7 +133,7 @@ def binding() -> dict[str, object]: "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, From 2a6e65a739b79a8232e1177610d11928fe92e26c Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Wed, 21 Oct 2026 14:47:30 -0400 Subject: [PATCH 11/17] Re-derive the merged tree's bundle digest and F0 attestation (#1073) Modal run 20261021T183247Z-97bb3af92 regenerated the bundle mirrors on the merge commit (no change) and re-derived every identity value there: the field-usage counts and claim pins, the ledger counts, the inventory digests and counts, and the pointer-inventory digest all match the committed values. The only moved value is the bundle spec_sha256, now 90fa3260... (main's kernel changes plus this branch's stage), which the pool tool pins. tools/spec_engine_coverage.py regenerated the attestation and --check reported 42329/42329 configuration fields and 41/41 inventory checks. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/evidence/spec-engine/us-f0-coverage.json | 66 +++++++++---------- .../us/test_us_multispine_pool_tool.py | 2 +- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 52fe0adb4f..2067827b8f 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "bd20f447db53159f719eba20c13a44efb825dc40fcc1db1d4f661ebbd6a7916b", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32545, "claim_count": 49, "claims": [ { @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 316, + "pointer_sha256": "0212e52e2bf1e5996fe6b4785b2d74f2fb5680ec38b258c6ad4dbeadbd181a1b", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1716, - "pointer_sha256": "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + "pointer_count": 1845, + "pointer_sha256": "4142b1037cddd9621b70eb9677be89510474475d9a705f1cedf7a32d7bf64db8", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -705,8 +705,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 112, - "pointer_sha256": "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", + "pointer_count": 120, + "pointer_sha256": "825b4ab09602353fc4d6c24b6f7654f7582502c7acf963b7ff6c117b07208eb4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1vintages.yaml", @@ -764,29 +764,29 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 63, - "pointer_sha256": "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", + "pointer_count": 67, + "pointer_sha256": "3f49902e745368ce8df0ad338a1697b76fd3299d35959977d36a8c09690ffb10", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/vintage_authorities", "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42329, + "consumed_field_count": 42329, "generation0_effect_counts": { - "legacy_behavior": 38498, - "no_generation0_effect": 3686 + "legacy_behavior": 38631, + "no_generation0_effect": 3698 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27729, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14010 + "legacy_behavior": 14143 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", - "resolved_binding_field_count": 9780, + "pointer_inventory_sha256": "37ce78d62b0eb4761eec78a22da637fc242c8adc516bb69ab6154430051c33e1", + "resolved_binding_field_count": 9784, "unused_field_count": 0 }, "inventory_coverage": { @@ -814,7 +814,7 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -824,7 +824,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", + "terminal_fields": 938 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", + "terminal_fields": 938 }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", "stages": 37 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "6fcc5e3cd9b9a4063ba211cfc499321b0e84807a60e28f76492b4e16a8a15678" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "19460fc168a6f9ed44e580d0e36777cce78bb33cb52b75630ee86203fa795684" + "spec_sha256": "90fa32603241f482d3457982f0467742e1917133d2b2a7fc120911d1ab7fcd32" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "19460fc168a6f9ed44e580d0e36777cce78bb33cb52b75630ee86203fa795684" + "spec_sha256": "90fa32603241f482d3457982f0467742e1917133d2b2a7fc120911d1ab7fcd32" }, "status": "pass" } diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 885b6b51a6..a0e723c228 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -345,7 +345,7 @@ def capture_equality(expected: object, actual: object) -> None: # fit_draw_next's caller-owned uniforms (microcosm#1332) move it. # microcosm#1073's scf_wealth education-debt declaration moves it # through the bundle's source stages. - "spec_sha256": "19460fc168a6f9ed44e580d0e36777cce78bb33cb52b75630ee86203fa795684", + "spec_sha256": "90fa32603241f482d3457982f0467742e1917133d2b2a7fc120911d1ab7fcd32", }, } From e6070ae528c27b9e38c3cdcd7c6c643b40fc484a Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Thu, 22 Oct 2026 11:39:17 -0400 Subject: [PATCH 12/17] Pin the #1073 fit receipt to the merged commit Reran build_1073_scf_education_debt_receipt.py on Modal at 2a6e65a73 (run 20261022T153020Z-2a6e65a73), after the merge with main brought in #1332's QRF change. Every fit figure is unchanged; only the receipt's commit, run and date move. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-education-debt-receipts.md | 2 +- .../receipts/1073-scf-education-debt-fit.json | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/experiments/1073-scf-education-debt-receipts.md b/experiments/1073-scf-education-debt-receipts.md index 45603c76a0..cbc6d6a5e4 100644 --- a/experiments/1073-scf-education-debt-receipts.md +++ b/experiments/1073-scf-education-debt-receipts.md @@ -1,6 +1,6 @@ # microcosm#1073 receipts: SCF 2025 federal student loan balances -Generated 2026-10-21 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`, commit `e35c4b0f911883585c150d2d6417dc01da2bb9d9`, Modal run `20261021T133110Z-e35c4b0f9` (`experiments/modal_1073.py`). Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. +Generated 2026-10-22 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`, commit `2a6e65a739b79a8232e1177610d11928fe92e26c`, Modal run `20261022T153020Z-2a6e65a73` (`experiments/modal_1073.py`). Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. Host columns impute the published default `populace-us-2024-spm-20260915` at its calibrated weights under three settings: the first receipt's construction (oldest adult, `A_MARITL` marriage, own children, whole-household income, every candidate predictor); the SCF-matching codings with every candidate predictor; and the build, which also leaves out interest and dividends and Social Security and pensions. The cause and the evidence are in the stage notes and `experiments/receipts/1073-predictor-shift-diagnosis.json`. diff --git a/experiments/receipts/1073-scf-education-debt-fit.json b/experiments/receipts/1073-scf-education-debt-fit.json index 2297b236c2..bced2dbead 100644 --- a/experiments/receipts/1073-scf-education-debt-fit.json +++ b/experiments/receipts/1073-scf-education-debt-fit.json @@ -1,10 +1,10 @@ { "schema_version": 1, "issue": "PolicyEngine/microcosm#1073", - "generated_at": "2026-10-21", + "generated_at": "2026-10-22", "provenance": { - "commit": "e35c4b0f911883585c150d2d6417dc01da2bb9d9", - "run": "20261021T133110Z-e35c4b0f9" + "commit": "2a6e65a739b79a8232e1177610d11928fe92e26c", + "run": "20261022T153020Z-2a6e65a73" }, "scf_files": { "rscfp2025.dta": { @@ -179,7 +179,7 @@ "50_to_61": 0.1615926964748789, "62_and_older": 0.06902425352362417 }, - "graduate_share_of_own_original_principal": 0.4773824150738412, + "graduate_share_of_own_original_principal": 0.47738241507384116, "ibr_new_borrower_share": 0.5915143468879298 } }, @@ -332,7 +332,7 @@ "50_to_61": 0.21093410606503074, "62_and_older": 0.11268289035889188 }, - "graduate_share_of_own_original_principal": 0.45213865186363184, + "graduate_share_of_own_original_principal": 0.4521386518636319, "ibr_new_borrower_share": 0.5904131759381358 } }, @@ -655,7 +655,7 @@ "50_to_61": 0.15779746384936916, "62_and_older": 0.06408599384077752 }, - "graduate_share_of_own_original_principal": 0.4619857518712365, + "graduate_share_of_own_original_principal": 0.46198575187123647, "ibr_new_borrower_share": 0.5919421297398388 } }, @@ -852,8 +852,8 @@ "mean_among_borrowers": 38129.7481875102, "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 918653496595.8906, - "parent_plus_share_of_total": 0.12228737578012734 + "federal_education_debt_total": 918653496595.8905, + "parent_plus_share_of_total": 0.12228737578012737 }, "household_balance_bands": { "under_5k": 0.09079966274592437, @@ -978,7 +978,7 @@ "50_to_61": 0.15141879980849401, "62_and_older": 0.06556996190233892 }, - "graduate_share_of_own_original_principal": 0.45012054963197445, + "graduate_share_of_own_original_principal": 0.4501205496319744, "ibr_new_borrower_share": 0.5931987024130821 } }, @@ -1139,7 +1139,7 @@ "50_to_61": 0.14875323302634258, "62_and_older": 0.06483235002890703 }, - "graduate_share_of_own_original_principal": 0.45388442861260153, + "graduate_share_of_own_original_principal": 0.4538844286126015, "ibr_new_borrower_share": 0.5880863294842776 } }, @@ -1645,7 +1645,7 @@ "50_to_61": 0.20778027670943267, "62_and_older": 0.07240287305689257 }, - "graduate_share_of_own_original_principal": 0.48269344604304165, + "graduate_share_of_own_original_principal": 0.4826934460430418, "ibr_new_borrower_share": 0.6147006160485154 } }, @@ -1833,7 +1833,7 @@ "50_to_61": 0.20778027670943267, "62_and_older": 0.07240287305689257 }, - "graduate_share_of_own_original_principal": 0.48269344604304165, + "graduate_share_of_own_original_principal": 0.4826934460430418, "ibr_new_borrower_share": 0.6147006160485154 } }, From 3f47edf234ad9e7f55a417b08b534a4c1a9dd4f4 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Thu, 22 Oct 2026 13:12:52 -0400 Subject: [PATCH 13/17] Describe #1073 as the student debt pilot in the changelog and module docstring Co-Authored-By: Claude Opus 5.5 (1M context) --- changelog.d/1073-scf-2025-education-debt.added.md | 2 +- .../src/microcosm/build/us_runtime/scf_education_debt.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/1073-scf-2025-education-debt.added.md b/changelog.d/1073-scf-2025-education-debt.added.md index 3b845e931f..dca3936e92 100644 --- a/changelog.d/1073-scf-2025-education-debt.added.md +++ b/changelog.d/1073-scf-2025-education-debt.added.md @@ -1,4 +1,4 @@ -The US build can impute federal student loan balances from the 2025 Survey of Consumer Finances (microcosm#1073, the EqRC student debt pilot). A new runtime module, `scf_education_debt`, writes four person inputs: `federal_student_loan_balance` and `federal_parent_plus_loan_balance` (policyengine-us#9721), and `is_income_based_repayment_new_borrower` and `federal_student_loan_graduate_share` (policyengine-us#9724). +The US build can impute federal student loan balances from the 2025 Survey of Consumer Finances (microcosm#1073, the student debt pilot). A new runtime module, `scf_education_debt`, writes four person inputs: `federal_student_loan_balance` and `federal_parent_plus_loan_balance` (policyengine-us#9721), and `is_income_based_repayment_new_borrower` and `federal_student_loan_graduate_share` (policyengine-us#9724). Only education debt reads the 2025 SCF. Net worth, the liquid-asset leaves and auto loans stay on the 2022 SCF. The `scf_wealth` stage declares the two 2025 files (`scfp2025s.zip`, `scf2025s.zip`), pinned by zip and member SHA-256 against a fresh Federal Reserve download, as artifacts 3 and 4, and a `scf_2025` vintage points at them. Moving the whole stage to 2025 later changes pinned coordinates, not code: the 2025 summary extract carries every column the shared predictor construction reads. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py index e86e7480a7..77fc0dc1a1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -1,7 +1,7 @@ """Federal student loan balances from the 2025 Survey of Consumer Finances. -microcosm#1073 carries SCF education debt into the US build for the EqRC student -debt pilot (PolicyEngine/us-student-debt#1). It writes four person inputs of +microcosm#1073 carries SCF education debt into the US build for the student debt +pilot (PolicyEngine/us-student-debt#1). It writes four person inputs of PolicyEngine-US: ``federal_student_loan_balance`` and ``federal_parent_plus_loan_balance`` (policyengine-us#9721), and ``is_income_based_repayment_new_borrower`` and From 04e82702703edf42e2ba9e7bc1dc590551925258 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Fri, 23 Oct 2026 10:23:06 -0400 Subject: [PATCH 14/17] Route loans for a child's education to co-resident adult children and swap X8000 codes on loans 4 and 6 (#1073) Parent PLUS is drawn for itself before the own shares, the IBR new-borrower share is labelled an upper bound, and the receipt benchmarks Parent PLUS and borrowers against Federal Student Aid. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-2025-education-debt.added.md | 4 +- .../build_1073_scf_education_debt_receipt.py | 482 +++++++++++- .../src/microcosm/build/us/source_stages.json | 54 +- .../build/us_runtime/scf_education_debt.py | 692 ++++++++++++++---- .../us/test_us_bundle_core_contracts.py | 2 +- .../us/test_us_scf_education_debt.py | 357 ++++++++- .../microcosm_build/us_spec_bundle.py | 2 +- tools/generate_us_bundle_from_constants.py | 2 +- 8 files changed, 1402 insertions(+), 193 deletions(-) diff --git a/changelog.d/1073-scf-2025-education-debt.added.md b/changelog.d/1073-scf-2025-education-debt.added.md index dca3936e92..21756d2721 100644 --- a/changelog.d/1073-scf-2025-education-debt.added.md +++ b/changelog.d/1073-scf-2025-education-debt.added.md @@ -2,9 +2,9 @@ The US build can impute federal student loan balances from the 2025 Survey of Co Only education debt reads the 2025 SCF. Net worth, the liquid-asset leaves and auto loans stay on the 2022 SCF. The `scf_wealth` stage declares the two 2025 files (`scfp2025s.zip`, `scf2025s.zip`), pinned by zip and member SHA-256 against a fresh Federal Reserve download, as artifacts 3 and 4, and a `scf_2025` vintage points at them. Moving the whole stage to 2025 later changes pinned coordinates, not code: the 2025 summary extract carries every column the shared predictor construction reads. -The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`; loans for a child go to the reference person's `federal_parent_plus_loan_balance`; codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. A person is an IBR new borrower when every federal loan they hold was borrowed in 2014 or later, and the graduate share weights each own loan's program by the amount borrowed. +The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`, and codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. In families where the public file reversed respondent and spouse (X8000 = 1), the codes of loans 4 and 6 were left unreversed (none of 54 degree-discriminating loans match the coded person, against all 25 in other families), so the stage swaps them back. A loan for a child's education is the co-resident child's own loan when the household listing has a child aged 18 or over who could have been in school when it was borrowed, unless only a parent could have borrowed it: an undergraduate loan above the $31,000 limit on a dependent student's own Direct Loans, borrowed while the child could still have been dependent. Graduate loans and loans borrowed after the child turned 24 are never Parent PLUS. Every other loan for a child is the reference person's `federal_parent_plus_loan_balance`. This takes the SCF's Parent PLUS share of federal balances from 13.8% to about 9%, against 6.7% (a floor) in Federal Student Aid's portfolio. A person is an IBR new borrower when every federal loan they hold, Parent PLUS included, was borrowed in 2014 or later; that share is an upper bound, since paid-off pre-2014 loans are not observed and a consolidated loan is dated by its newest underlying loan. The graduate share weights each own loan's program by the amount borrowed. -The model. One weighted QRF draws each household's federal education debt. A second, fit on borrower families and conditioning on that total, draws the holder shares, graduate shares and new-borrower flags. Both condition on the reference person's age, sex and race, a spouse or partner being present, the children of the reference person or partner, and the wages and income less capital gains of the reference person, spouse or partner and their children. The CPS side codes each predictor the way the SCF defines it: SCF `MARRIED` counts cohabiting partners, SCF `KIDS` counts the partner's children, the reference person comes from the householder couple (the man, or the older partner), and income covers the primary economic unit rather than the whole household. The shared interest-dividend and Social Security-pension predictors are left out (`SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS`). The CPS host carries tax-record imputations of both, so at reference ages 25-34 it reports them for 42% and 18% of households against 16% and 6% of SCF families, and that shift was the main reason the first host fit drew too few borrowers. The reference person and the member its `A_SPOUSE` (else `PECOHAB`) pointer names receive the balances. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. +The model. One weighted QRF draws each household's federal education debt. A second, fit on borrower families and conditioning on that total and the number of adult children, draws the Parent PLUS, adult child's, reference person's and spouse's shares in that order, normalized to sum to one, then the graduate shares and new-borrower flags. Parent PLUS is drawn for itself, so a family like the donors without it rarely draws it. Both condition on the reference person's age, sex and race, a spouse or partner being present, the children of the reference person or partner, and the wages and income less capital gains of the reference person, spouse or partner and their children. The CPS side codes each predictor the way the SCF defines it: SCF `MARRIED` counts cohabiting partners, SCF `KIDS` counts the partner's children, the reference person comes from the householder couple (the man, or the older partner), and income covers the primary economic unit rather than the whole household. The shared interest-dividend and Social Security-pension predictors are left out (`SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS`). The CPS host carries tax-record imputations of both, so at reference ages 25-34 it reports them for 42% and 18% of households against 16% and 6% of SCF families, and that shift was the main reason the first host fit drew too few borrowers. The reference person, the member its `A_SPOUSE` (else `PECOHAB`) pointer names, and the youngest child of either aged 18 or over receive the balances; a child's draw in a household without an adult child is the reference person's Parent PLUS loan. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. The fiscal-refresh release tool runs the stage only with `--scf-education-debt`, because the pinned policyengine-us does not define the four inputs yet. The four columns join `US_STORED_NON_VARIABLE_COLUMNS` with the pending-engine reason the parent ids carry, and leave it when the engine pin moves. They are not in the release input-coverage manifest, whose check requires every column to be an engine input. No release is certified with them. diff --git a/experiments/build_1073_scf_education_debt_receipt.py b/experiments/build_1073_scf_education_debt_receipt.py index 186eff7186..bf2bd7354b 100644 --- a/experiments/build_1073_scf_education_debt_receipt.py +++ b/experiments/build_1073_scf_education_debt_receipt.py @@ -5,8 +5,10 @@ 1. **Source checks.** The loader's education debt against the Fed's published Alternate Table 13 (19.92% of families hold education debt, median - $25,500), and how the federal flag, X7179 and the whose-education codes - split the balances the X7179 and holder rules act on. + $25,500), how the federal flag, X7179 and the whose-education codes split + the balances the X7179 and holder rules act on, the X8000 degree test for + loans 4 and 6, and the Parent PLUS share and borrowers under each reading + of a loan for a child's education. 2. **Hold-out fit and predictor ablation.** Families are split into five folds by ``yy1`` (all implicates of a family stay together). Each fold is drawn from the stage's two QRFs fit on the other four, and the drawn table is @@ -18,7 +20,9 @@ predictor, and the build (that definition without the two shifted income predictors). -Nothing here is a gate or a calibration target. The committed receipts are +Federal Student Aid's portfolio tables (``FSA``) are the benchmark for the +Parent PLUS share, borrower counts and borrower bands. Nothing here is a gate +or a calibration target. The committed receipts are produced on Modal by ``experiments/modal_1073.py`` (step ``receipt``), which clones a pushed commit and verifies every input's SHA-256; the receipt records that commit and the run id. A local run needs about 4 GB: @@ -41,9 +45,11 @@ import numpy as np import pandas as pd +import microcosm.build.us_runtime.scf_education_debt as education_debt from microcosm.build.us_runtime.scf_education_debt import ( SCF_2025_FULL_EXTRACT, SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, SCF_EDUCATION_DEBT_BALANCE_BANDS, SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, @@ -51,13 +57,20 @@ SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, SCF_EDUCATION_DEBT_TOTAL_TARGET, + SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, SCF_EDUCATION_LOAN_BALANCE_COLUMNS, SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + SCF_EDUCATION_LOAN_YEAR_COLUMNS, + SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + SCF_REVERSED_COLUMN, RecipientDefinition, ScfPublicFile, _read_stata_columns, + derive_scf_education_debt_targets, draw_scf_education_debt_households, fetch_scf_public_file, impute_us_scf_education_debt, @@ -79,6 +92,71 @@ "share_of_families_with_education_debt_pct": 19.92, "median_among_holders": 25_500.0, } +#: Federal Student Aid portfolio tables, as published (the chronicle#347 source +#: packages). Downloaded from +#: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/; +#: the SHA-256 of each file read is listed. Recipients are students, so a +#: Parent PLUS loan counts the child; the by-age and by-debt-size tables count +#: borrowers, so they count the parent. +FSA = { + "files": { + "PortfolioSummary.xls": ( + "11606d4b93ceea969fcae0f790f3a4e1c2999b62050559ce89bdc269810a62ee" + ), + "PortfoliobyLoanType.xls": ( + "e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33" + ), + "Portfolio-by-Age.xls": ( + "c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef" + ), + "Portfolio-by-Debt-Size.xls": ( + "881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d" + ), + }, + "total_dollars_fy2026_q3": 1_724.5e9, + "unduplicated_recipients_fy2026_q3": 42.3e6, + "parent_plus_dollars_fy2026_q3": 115.0e9, + "parent_plus_recipients_fy2026_q3": 3.5e6, + "borrowers_by_age_fy2026_q2": { + "24_and_younger": 6.2e6, + "25_to_34": 14.0e6, + "35_to_49": 15.0e6, + "50_to_61": 6.4e6, + "62_and_older": 3.2e6, + "not_reported": 0.1e6, + }, + "borrowers_by_debt_size_fy2026_q2": { + "under_5k": 6.9e6, + "5k_to_10k": 7.2e6, + "10k_to_20k": 9.2e6, + "20k_to_40k": 9.5e6, + "40k_to_60k": 4.2e6, + "60k_to_80k": 2.6e6, + "80k_to_100k": 1.4e6, + "100k_to_200k": 2.6e6, + "200k_plus": 1.2e6, + }, +} +#: The build's host fit in the previous receipt (before loans for a child's +#: education were routed to co-resident children and Parent PLUS was drawn +#: for itself), for the before/after table. The SCF's own "before" column is +#: recomputed here from the old rule. +PREVIOUS_RECEIPT = { + "commit": "2a6e65a739b79a8232e1177610d11928fe92e26c", + "run": "20261022T153020Z-2a6e65a73", + "host_build": { + "parent_plus_share_of_total": 0.16758194869071996, + "federal_education_debt_total": 858421208954.9309, + "borrower_weight": 21223403.433088087, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257, + }, + }, +} #: The published default the host fit imputes onto, and its release-manifest #: SHA-256 (releases/populace-us-2024-spm-20260915/release_manifest.json). DEFAULT_HOST_RELEASE = "populace-us-2024-spm-20260915" @@ -264,6 +342,187 @@ def source_checks(donor: pd.DataFrame, full: pd.DataFrame) -> dict[str, object]: } +_COMPLETED_COLUMNS = ("x7881", "x7886", "x7891", "x7896", "x7901", "x7996") +_CONSOLIDATED_COLUMNS = ("x9320", "x9321", "x9322", "x9323", "x9324", "x9325") +_DEGREE_LEVEL = {10: 1, 11: 1, 12: 2, 13: 3, 14: 4, 15: 4} +FULL_FILE_RECEIPT_COLUMNS = ( + "y1", + "yy1", + "x42001", + "x5931", + "x6111", + SCF_REVERSED_COLUMN, + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + *SCF_EDUCATION_LOAN_YEAR_COLUMNS, + *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + *_COMPLETED_COLUMNS, + *_CONSOLIDATED_COLUMNS, + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, +) + + +def _matrix(full: pd.DataFrame, columns: Sequence[str]) -> np.ndarray: + return np.column_stack( + [pd.to_numeric(full[c], errors="coerce").fillna(0).to_numpy() for c in columns] + ) + + +def x8000_degree_check(full: pd.DataFrame) -> list[dict[str, object]]: + """Do loans' whose-education codes match the coded person's degree? + + Completed loans coded 1 or 2 that financed a degree (programs 10-15), in + families with a spouse or partner (X6111 recorded). A loan is consistent + with a person whose highest degree (X5931 for the reference person, X6111 + for the spouse or partner) is at least the program's level; it + discriminates when it is consistent with exactly one of them. Raw codes, + before the stage's correction; counts pool the five implicates. + """ + + level = np.vectorize(lambda code: _DEGREE_LEVEL.get(int(code), 0)) + balance = _matrix(full, SCF_EDUCATION_LOAN_BALANCE_COLUMNS) + whose = _matrix(full, SCF_EDUCATION_LOAN_WHOSE_COLUMNS) + completed = _matrix(full, _COMPLETED_COLUMNS) + program_level = level(_matrix(full, SCF_EDUCATION_LOAN_PROGRAM_COLUMNS)) + reference = level(pd.to_numeric(full["x5931"]).fillna(0).to_numpy()) + spouse_degree = pd.to_numeric(full["x6111"]).fillna(0).to_numpy() + spouse = level(spouse_degree) + reversed_family = pd.to_numeric(full[SCF_REVERSED_COLUMN]).to_numpy() == 1 + implicate = pd.to_numeric(full["y1"]).astype(int).to_numpy() % 10 + rows = [] + for loans, positions in (("1, 2, 3, 5", (0, 1, 2, 4)), ("4, 6", (3, 5))): + for family, label in ((False, "not reversed"), (True, "reversed")): + for code in (1, 2): + counts = {"all": [0, 0], "implicate_1": [0, 0]} + for position in positions: + loan = ( + (balance[:, position] > 0) + & (whose[:, position] == code) + & (completed[:, position] == 1) + & (program_level[:, position] > 0) + & (spouse_degree != 0) + & (reversed_family == family) + ) + fits_reference = reference >= program_level[:, position] + fits_spouse = spouse >= program_level[:, position] + discriminating = loan & (fits_reference != fits_spouse) + coded = fits_reference if code == 1 else fits_spouse + for key, mask in ( + ("all", discriminating), + ("implicate_1", discriminating & (implicate == 1)), + ): + counts[key][0] += int(mask.sum()) + counts[key][1] += int((mask & coded).sum()) + rows.append( + { + "loans": loans, + "x8000": label, + "code": code, + "discriminating": counts["all"][0], + "consistent_with_coded_person": counts["all"][1], + "discriminating_implicate_1": counts["implicate_1"][0], + "consistent_implicate_1": counts["implicate_1"][1], + } + ) + return rows + + +def _derive_reading(full: pd.DataFrame, reading: str) -> pd.DataFrame: + """Donor targets under one reading of the whose-education codes. + + ``before`` is the previous rule: no X8000 correction and every loan for a + child's education the head's Parent PLUS loan (as if no child were + listed). ``x8000_only`` adds the correction. ``build`` is the stage. + ``within_limit_parent_plus`` reads an undergraduate loan within the + dependent limit for a child of dependent age as Parent PLUS too. + """ + + frame = full.copy() + if reading in {"before", "x8000_only"}: + for column in SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS: + frame[column] = 0 + if reading == "before": + frame[SCF_REVERSED_COLUMN] = 5 + if reading != "within_limit_parent_plus": + return derive_scf_education_debt_targets(frame) + limit = education_debt._DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT + education_debt._DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT = -1.0 + try: + return derive_scf_education_debt_targets(frame) + finally: + education_debt._DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT = limit + + +READINGS = ("before", "x8000_only", "build", "within_limit_parent_plus") + + +def child_education_readings(full: pd.DataFrame) -> dict[str, dict[str, float]]: + """Parent PLUS share and borrowers (people) under each reading.""" + + weight = full["wgt"].to_numpy(dtype=np.float64) + result = {} + for reading in READINGS: + targets = _derive_reading(full, reading) + total = float(np.dot(targets[SCF_EDUCATION_DEBT_TOTAL_TARGET], weight)) + plus = targets["federal_parent_plus_loan_balance"].to_numpy() + head = targets["head_federal_student_loan_balance"].to_numpy() + plus + spouse = targets["spouse_federal_student_loan_balance"].to_numpy() + child = targets["child_federal_student_loan_balance"].to_numpy() + result[reading] = { + "parent_plus_share_of_federal": float(np.dot(plus, weight)) / total, + "child_own_share_of_federal": float(np.dot(child, weight)) / total, + "parent_plus_dollars": float(np.dot(plus, weight)), + "borrowers": float( + weight[head > 0].sum() + + weight[spouse > 0].sum() + + weight[child > 0].sum() + ), + "parent_plus_holders": float(weight[plus > 0].sum()), + "child_borrowers": float(weight[child > 0].sum()), + } + return result + + +def consolidated_share_of_new_loans(full: pd.DataFrame) -> float: + """Share of federal balances dated 2014 or later on a consolidated loan.""" + + weight = full["wgt"].to_numpy(dtype=np.float64)[:, None] + balance = _matrix(full, SCF_EDUCATION_LOAN_BALANCE_COLUMNS) + federal = (_matrix(full, SCF_EDUCATION_LOAN_FEDERAL_COLUMNS) == 1) & (balance > 0) + new = _matrix(full, SCF_EDUCATION_LOAN_YEAR_COLUMNS) >= 2014 + consolidated = _matrix(full, _CONSOLIDATED_COLUMNS) == 1 + dated_new = np.where(federal & new, balance, 0.0) * weight + return float(np.where(consolidated, dated_new, 0.0).sum() / dated_new.sum()) + + +def donor_under_reading( + donor: pd.DataFrame, full: pd.DataFrame, reading: str +) -> pd.DataFrame: + """The donor with its targets derived under another reading.""" + + targets = _derive_reading(full, reading) + keys = pd.MultiIndex.from_arrays( + [ + pd.to_numeric(full["yy1"]).to_numpy(), + pd.to_numeric(full["y1"]).astype(int).to_numpy() % 10, + ] + ) + targets.index = keys + aligned = targets.reindex( + pd.MultiIndex.from_arrays( + [donor["yy1"].to_numpy(), donor["implicate"].to_numpy()] + ) + ) + result = donor.copy() + for column in targets.columns: + result[column] = aligned[column].to_numpy() + return result + + def _fold_of_family(families: np.ndarray, *, folds: int, seed: int) -> np.ndarray: unique = np.unique(families) rng = np.random.default_rng(np.random.SeedSequence([seed, 1073, 5])) @@ -286,7 +545,9 @@ def holdout_draws( for k in range(folds): held_out = fold == k drawn = draw_scf_education_debt_households( - donor.loc[held_out, list(predictors)], + donor.loc[ + held_out, [*predictors, SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] + ], donor.loc[~held_out], seed=seed + k, predictors=predictors, @@ -464,8 +725,10 @@ def row(label: str, getter, formatter) -> str: "setting's income unit, the income concept that setting conditions on. " "Borrower figures count " "people with a federal balance (own plus Parent PLUS), in Federal Student " - "Aid's balance and age bands; SCF borrowers are the reference person and " - "spouse. Nothing here is a gate or a calibration target.", + "Aid's balance and age bands; SCF borrowers are the reference person, the " + "spouse or partner and an adult child holding a loan for their own " + "education, whose public-file age band (18-27 reads 25) is spread evenly " + "over its years. Nothing here is a gate or a calibration target.", "", "## Source checks", "", @@ -488,11 +751,163 @@ def row(label: str, getter, formatter) -> str: lines.append(f"| {key.replace('_', ' ')} | {_pct(value)} |") lines += [ "", - "| Federal balances on loans 1-6, by whose education | Share |", + "| Federal balances on loans 1-6, by whose education (codes as published) " + "| Share |", "|---|---:|", ] for key, value in checks["federal_balance_share_by_whose_education_code"].items(): lines.append(f"| {key} | {_pct(value)} |") + lines += [ + "", + "### X8000: are the whose-education codes reversed?", + "", + "Completed degree loans coded 1 or 2 in families with a spouse or partner. " + "A loan discriminates when it is consistent with exactly one partner's " + "highest degree (X5931, X6111: at least the program's level); the last " + "columns count those consistent with the coded person (code 1 the " + "reference person, code 2 the spouse or partner), as published, before " + "the stage swaps codes 1 and 2 of loans 4 and 6 in reversed families. " + "Counts pool the five implicates (implicate 1 in brackets).", + "", + "| Loans | X8000 | Code | Discriminating | Consistent with the coded person |", + "|---|---|---:|---:|---:|", + ] + for entry in checks["x8000_degree_check"]: + lines.append( + f"| {entry['loans']} | {entry['x8000']} | {entry['code']} | " + f"{entry['discriminating']} ({entry['discriminating_implicate_1']}) | " + f"{entry['consistent_with_coded_person']} " + f"({entry['consistent_implicate_1']}) |" + ) + readings = checks["child_education_readings"] + reading_labels = { + "before": "Before: every code-3 loan Parent PLUS, codes as published", + "x8000_only": "X8000 correction only", + "build": "Build: co-resident adult child unless only a parent could borrow", + "within_limit_parent_plus": ( + "Alternative: within-limit undergraduate loans for a dependent-age " + "child also Parent PLUS" + ), + } + lines += [ + "", + "### Loans for a child's education (SCF 2025, weighted)", + "", + "FSA's Direct Parent PLUS balance is " + f"{_pct(FSA['parent_plus_dollars_fy2026_q3'] / FSA['total_dollars_fy2026_q3'])}" + " of its portfolio at FY2026 Q3, a floor, since consolidated Parent PLUS " + "loans sit in the consolidation total.", + "", + "| Reading | Parent PLUS share | Child's own share | Parent PLUS holders " + "| Child borrowers | Borrowers |", + "|---|---:|---:|---:|---:|---:|", + ] + for reading, label in reading_labels.items(): + entry = readings[reading] + lines.append( + f"| {label} | {_pct(entry['parent_plus_share_of_federal'])} | " + f"{_pct(entry['child_own_share_of_federal'])} | " + f"{entry['parent_plus_holders'] / 1e6:,.1f}M | " + f"{entry['child_borrowers'] / 1e6:,.1f}M | " + f"{entry['borrowers'] / 1e6:,.1f}M |" + ) + lines += [ + "", + "IBR new-borrower status is an upper bound: the SCF sees only loans still " + "owed, so paid-off pre-2014 loans are missed, and a consolidated loan is " + "dated by its most recent underlying loan. " + f"{_pct(checks['consolidated_share_of_federal_balances_dated_2014_or_later'])}" + " of federal balances dated 2014 or later are on a loan that has been " + "consolidated or refinanced (X9320-X9325).", + ] + before = receipt["scf_2025_observed_before"] + previous = receipt["previous_receipt"]["host_build"] + build = settings.get(BUILD_SETTING, {}).get("fit") + by_age_fsa = { + k: v + for k, v in FSA["borrowers_by_age_fy2026_q2"].items() + if k != "not_reported" + } + fsa_age_total = sum(by_age_fsa.values()) + fsa_borrowers = sum(FSA["borrowers_by_age_fy2026_q2"].values()) + compare = [ + ("SCF 2025, before", { + "pp": before["households"]["parent_plus_share_of_total"], + "pp_dollars": before["households"]["parent_plus_share_of_total"] + * before["households"]["federal_education_debt_total"], + "total": before["households"]["federal_education_debt_total"], + "borrowers": before["borrowers"]["borrower_weight"], + "ages": before["borrowers"]["age_bands"], + }), + ("SCF 2025", { + "pp": observed["households"]["parent_plus_share_of_total"], + "pp_dollars": observed["households"]["parent_plus_share_of_total"] + * observed["households"]["federal_education_debt_total"], + "total": observed["households"]["federal_education_debt_total"], + "borrowers": observed["borrowers"]["borrower_weight"], + "ages": observed["borrowers"]["age_bands"], + }), + ("Host: Build, before", { + "pp": previous["parent_plus_share_of_total"], + "pp_dollars": previous["parent_plus_share_of_total"] + * previous["federal_education_debt_total"], + "total": previous["federal_education_debt_total"], + "borrowers": previous["borrower_weight"], + "ages": previous["age_bands"], + }), + ] # fmt: skip + if build is not None: + compare.append( + ( + "Host: Build", + { + "pp": build["households"]["parent_plus_share_of_total"], + "pp_dollars": build["households"]["parent_plus_share_of_total"] + * build["households"]["federal_education_debt_total"], + "total": build["households"]["federal_education_debt_total"], + "borrowers": build["borrowers"]["borrower_weight"], + "ages": build["borrowers"]["age_bands"], + }, + ) + ) + lines += [ + "", + "## Parent PLUS and borrowers against Federal Student Aid", + "", + "Before is the previous rule (every loan for a child's education the " + "head's Parent PLUS loan, the residual of the composition draw); the " + "host's before column is the build in the previous receipt (commit " + f"`{receipt['previous_receipt']['commit'][:9]}`, Modal run " + f"`{receipt['previous_receipt']['run']}`). FSA's Parent PLUS share is a " + "floor; its dollars are the whole federal portfolio, of which household " + "surveys capture part, so the shares compare and the levels do not. FSA " + "borrowers come from its by-age table (FY2026 Q2), which counts the " + "parent on a Parent PLUS loan; its " + f"{FSA['unduplicated_recipients_fy2026_q3'] / 1e6:.1f}M unduplicated " + "recipients (FY2026 Q3) count the student instead.", + "", + "| | FSA | " + " | ".join(label for label, _ in compare) + " |", + "|---|---:|" + "---:|" * len(compare), + "| Parent PLUS share of federal balances | " + f"{_pct(FSA['parent_plus_dollars_fy2026_q3'] / FSA['total_dollars_fy2026_q3'])}" + " (floor) | " + " | ".join(_pct(c["pp"]) for _, c in compare) + " |", + f"| Parent PLUS balances | ${FSA['parent_plus_dollars_fy2026_q3'] / 1e9:,.1f}B | " + + " | ".join(f"${c['pp_dollars'] / 1e9:,.1f}B" for _, c in compare) + + " |", + f"| Federal balances | ${FSA['total_dollars_fy2026_q3'] / 1e9:,.1f}B | " + + " | ".join(f"${c['total'] / 1e9:,.1f}B" for _, c in compare) + + " |", + f"| Borrowers (people) | {fsa_borrowers / 1e6:,.1f}M | " + + " | ".join(f"{c['borrowers'] / 1e6:,.1f}M" for _, c in compare) + + " |", + ] + for label, _, _ in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: + lines.append( + f"| Borrowers aged {label.replace('_', ' ')} | " + f"{_pct(by_age_fsa[label] / fsa_age_total)} | " + + " | ".join(_pct(c["ages"][label]) for _, c in compare) + + " |" + ) lines += [ "", "## Fit", @@ -540,7 +955,7 @@ def row(label: str, getter, formatter) -> str: _dollars, ), row( - "IBR new borrowers among borrowers", + "IBR new borrowers among borrowers (upper bound)", lambda t: t["borrowers"]["ibr_new_borrower_share"], _pct, ), @@ -552,17 +967,35 @@ def row(label: str, getter, formatter) -> str: "", "### Borrowers by FSA balance band", "", - header, - rule, + "FSA: borrowers by debt size, FY2026 Q2.", + "", + header + " FSA |", + rule + "---:|", ] + by_size = FSA["borrowers_by_debt_size_fy2026_q2"] for label, _, _ in SCF_EDUCATION_DEBT_BALANCE_BANDS: lines.append( row(label, lambda t, k=label: t["borrowers"]["balance_bands"][k], _pct) + + f" {_pct(by_size[label] / sum(by_size.values()))} |" ) - lines += ["", "### Borrowers by FSA age band", "", header, rule] + by_age = { + k: v + for k, v in FSA["borrowers_by_age_fy2026_q2"].items() + if k != "not_reported" + } + lines += [ + "", + "### Borrowers by FSA age band", + "", + "FSA: borrowers by age, FY2026 Q2, of those with an age reported.", + "", + header + " FSA |", + rule + "---:|", + ] for label, _, _ in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: lines.append( row(label, lambda t, k=label: t["borrowers"]["age_bands"][k], _pct) + + f" {_pct(by_age[label] / sum(by_age.values()))} |" ) for section, title in ( ( @@ -724,20 +1157,13 @@ def main(argv: Sequence[str] | None = None) -> int: donor = load_scf_2025_education_debt_donor(summary_path, full_path) full = _read_stata_columns( - full_path, - ( - "y1", - "yy1", - "x42001", - *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, - SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, - *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, - *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, - ), - label="2025 full file", + full_path, FULL_FILE_RECEIPT_COLUMNS, label="2025 full file" ) full["wgt"] = full["x42001"] / 5.0 observed = scf_education_debt_donor_fit_table(donor) + observed_before = scf_education_debt_donor_fit_table( + donor_under_reading(donor, full, "before") + ) receipt: dict[str, object] = { "schema_version": 1, "issue": ISSUE, @@ -751,8 +1177,18 @@ def main(argv: Sequence[str] | None = None) -> int: } for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT) }, - "source_checks": source_checks(donor, full), + "source_checks": { + **source_checks(donor, full), + "x8000_degree_check": x8000_degree_check(full), + "child_education_readings": child_education_readings(full), + "consolidated_share_of_federal_balances_dated_2014_or_later": ( + consolidated_share_of_new_loans(full) + ), + }, + "fsa": FSA, + "previous_receipt": PREVIOUS_RECEIPT, "scf_2025_observed": observed, + "scf_2025_observed_before": observed_before, "holdout": { "folds": args.folds, "seed": args.seed, diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index f0caee74e4..71e8550d47 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1193,13 +1193,19 @@ "federal_education_debt", "head_federal_student_loan_balance", "spouse_federal_student_loan_balance", + "child_federal_student_loan_balance", "federal_parent_plus_loan_balance", + "parent_plus_share", + "child_own_share", "head_own_share", "spouse_own_share", "head_graduate_share", "spouse_graduate_share", + "child_graduate_share", "head_is_ibr_new_borrower", - "spouse_is_ibr_new_borrower" + "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower", + "adult_children_in_household" ], "method": "scf_federal_education_loans_by_holder", "inputs": [ @@ -1239,9 +1245,32 @@ "x7851", "x7905", "x7928", - "x7951" + "x7951", + "x8000", + "x108", + "x114", + "x120", + "x126", + "x132", + "x202", + "x208", + "x214", + "x220", + "x226", + "x110", + "x116", + "x122", + "x128", + "x134", + "x204", + "x210", + "x216", + "x222", + "x228" ], "allocation": "x7179_in_proportion_to_loans_1_6", + "whose_education_correction": "x8000_reversed_families_swap_codes_1_and_2_on_loans_4_and_6", + "child_education_holder": "co_resident_adult_child_unless_only_a_parent_could_borrow", "source": "scf_2025_full_extract" }, { @@ -1274,16 +1303,22 @@ "own_children_in_household", "employment_income", "household_income", + "adult_children_in_household", "federal_education_debt" ], "targets": [ + "parent_plus_share", + "child_own_share", "head_own_share", "spouse_own_share", "head_graduate_share", "spouse_graduate_share", + "child_graduate_share", "head_is_ibr_new_borrower", - "spouse_is_ibr_new_borrower" + "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower" ], + "holder_shares": "drawn_in_order_then_normalized_to_one", "weight": "wgt", "response_filter": "federal_education_debt > 0", "n_estimators": 100, @@ -1297,17 +1332,24 @@ "is_income_based_repayment_new_borrower", "federal_student_loan_graduate_share" ], - "method": "allocate_household_education_debt_to_reference_person_and_spouse", + "method": "allocate_household_education_debt_to_reference_person_spouse_and_adult_child", "inputs": [ "federal_education_debt", + "parent_plus_share", + "child_own_share", "head_own_share", "spouse_own_share", "head_graduate_share", "spouse_graduate_share", + "child_graduate_share", "head_is_ibr_new_borrower", "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower", "A_SPOUSE", - "PECOHAB" + "PECOHAB", + "PEPAR1", + "PEPAR2", + "age" ] }, { @@ -1358,7 +1400,7 @@ "federal_parent_plus_loan_balance", "federal_student_loan_graduate_share" ], - "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family's federal share and holder split of loans 1-6. Whose education: code 1 (the reference person) and code 2 (spouse or partner) are each person's own federal_student_loan_balance; code 3 (child, with grandchild and other relative folded in by the public file) is the reference person's federal_parent_plus_loan_balance; codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the reference person's own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower is true when every federal loan the person holds was borrowed in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person's own federal loans that financed a master's, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all families; a second, fit on borrower families and conditioning on the total, draws the chained holder shares, graduate shares and new-borrower flags. Both condition on the reference person's age, sex and race, a spouse or partner being present (SCF MARRIED counts partners), the children of the reference person or partner (SCF KIDS), and the wages and income less capital gains of the reference person, spouse or partner and their children (the CPS counterpart of the SCF primary economic unit). The shared interest-dividend and Social Security-pension predictors are left out: the CPS host carries tax-record imputations of both, so at reference ages 25-34 42% and 18% of host households report some against 16% and 6% of SCF families, and mapping them to the SCF distribution moves the host's borrower share most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). The reference person is the CPS householder or, with a spouse or partner present, the man of a mixed-sex couple or else the older partner; that person and the member named by its A_SPOUSE (else PECOHAB) pointer receive the balances, and without one present the spouse's draw goes to the reference person. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals." + "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family's federal share and holder split of loans 1-6. Whose education: code 1 (the reference person) and code 2 (spouse or partner) are each person's own federal_student_loan_balance. Where X8000 = 1 the public file reverses respondent and spouse/partner codes, but not those of loans 4 and 6 (X7893, X7993): completed degree loans consistent with exactly one partner's highest degree (X5931, X6111) match the coded person on loans 1, 2, 3 and 5 and on loans 4 and 6 of unreversed families (25 of 25), but on loans 4 and 6 of reversed families match in none of 54 (all implicates), so codes 1 and 2 of those two loans are swapped back in reversed families. Code 3 (child, with grandchild and other relative folded in by the public file) is a co-resident child's own federal_student_loan_balance when the household listing (X108-X226) has a child aged 18 or over who could have been in school in the year borrowed (16 or older then, at the oldest age of the public file's age band), unless only a parent could have borrowed it: an undergraduate or certificate loan above the $31,000 aggregate limit on a dependent undergraduate's own Direct Loans (34 CFR 685.203(e)(1)) borrowed while the child could still have been under 24 and so dependent (20 U.S.C. 1087vv(d)(1)(A)). Graduate loans and loans borrowed after the child turned 24 are never Parent PLUS, which finances only a dependent undergraduate (34 CFR 685.200(c)(2)(i)). Every other code-3 loan is the reference person's federal_parent_plus_loan_balance: the screener (X7801) leaves out loans of people living elsewhere, so a loan for a child not in the household is the parent's. Codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the reference person's own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower is true when every federal loan the person holds (for the reference person, Parent PLUS too, as 34 CFR 685.209(b)(13)(ii) looks at any Direct or FFEL balance) was borrowed in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014); the share is an upper bound, because paid-off pre-2014 loans are not observed and a consolidated loan is dated by its most recent underlying loan. federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person's own federal loans that financed a master's, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all families; a second, fit on borrower families and conditioning also on the total and the number of adult children, draws the Parent PLUS, adult child's, reference person's and spouse's shares in that order, normalized to sum to one, then the three graduate shares and new-borrower flags. Both condition on the reference person's age, sex and race, a spouse or partner being present (SCF MARRIED counts partners), the children of the reference person or partner (SCF KIDS), and the wages and income less capital gains of the reference person, spouse or partner and their children (the CPS counterpart of the SCF primary economic unit). The shared interest-dividend and Social Security-pension predictors are left out: the CPS host carries tax-record imputations of both, so at reference ages 25-34 42% and 18% of host households report some against 16% and 6% of SCF families, and mapping them to the SCF distribution moves the host's borrower share most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). The reference person is the CPS householder or, with a spouse or partner present, the man of a mixed-sex couple or else the older partner; that person, the member named by its A_SPOUSE (else PECOHAB) pointer, and the youngest child of either aged 18 or over (PEPAR1/PEPAR2) receive the balances. Without a spouse or partner present the spouse's draw goes to the reference person, and a child's draw in a household without an adult child is the reference person's Parent PLUS loan. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals." }, { "stage": "ssi_disability_criteria", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py index 77fc0dc1a1..83fd953ee4 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -36,21 +36,56 @@ in the same proportions as loans 1-6. A family with X7179 but no balance on loans 1-6 contributes nothing (none in the public files read). - **Whose-education rule** (:data:`SCF_WHOSE_EDUCATION_HOLDER`). Code 1 - (respondent; the public file swaps respondent and spouse codes so code 1 is - the SCF reference person) is the head's own loan, and code 2 (spouse or - partner) the spouse's own loan; both go to ``federal_student_loan_balance``. - Code 3 (child or stepchild; the public file folds grandchild and other - relative into 3) is a loan for a child's education and goes to the head's - ``federal_parent_plus_loan_balance``. Code 6 (multiple people), code 7 - (unrelated person) and code -7 (other) go to the head's own balance: none can - be a Parent PLUS loan for one dependent student, and the SCF reference person - is the household's default borrower. Any other code (the public file folds 4 - and 5 into 3, so none is expected) also goes to the head. + (respondent) is the head's own loan and code 2 (spouse or partner) the + spouse's; both go to ``federal_student_loan_balance``. Where X8000 = 1 the + public file reverses every respondent and spouse/partner variable so that + code 1 is the SCF reference person, but it left the codes of loans 4 and 6 + (X7893, X7993) unreversed. The test: a completed loan for a degree should + be consistent with the coded person's highest degree (X5931 for the + reference person, X6111 for the spouse or partner). Of the loans consistent + with exactly one partner's degree, loans 1, 2, 3 and 5 match the coded person + in reversed and other families alike, and loans 4 and 6 match in all 25 such + loans (five implicates) of families that were not reversed, but in none of + the 54 of reversed families (10 in implicate 1). So in a reversed family + codes 1 and 2 of loans 4 and 6 are swapped back + (:data:`SCF_WHOSE_EDUCATION_UNREVERSED_LOANS`); the receipt reruns the test. + The codebook names no such exception, so the correction is tied to the + pinned member SHA-256. Code 6 (multiple people), code 7 (unrelated person) + and code -7 (other) go to the head's own balance: none can be a Parent PLUS + loan for one dependent student, and the SCF reference person is the + household's default borrower. Any other code (the public file folds 4 and 5 + into 3, so none is expected) also goes to the head. +- **Loans for a child's education** (code 3, child or stepchild; the public + file folds grandchild and other relative into it). The survey asks about + the loans of the family living in the household, and the screener (X7801) + leaves out loans of people who live elsewhere, so a code-3 loan is a + co-resident child's own loan or the parent's Parent PLUS loan. It is the child's own + (``child_federal_student_loan_balance``) when the household listing + (persons 3-12, X108-X226) has a child (code 4) aged 18 or over who could + have been in school when it was borrowed (16 or older in that year, at the + oldest age the child's public-file age band allows: the file bands listed + ages, so 18-27 all read 25), unless only a parent could have borrowed it: + an undergraduate or certificate loan whose amount borrowed exceeds the + $31,000 aggregate limit on a dependent undergraduate's own Direct Loans + (34 CFR 685.203(e)(1)), borrowed while the child could still have been + under 24, and so dependent (20 U.S.C. 1087vv(d)(1)(A)). Parent PLUS + finances only a dependent undergraduate (34 CFR 685.200(c)(2)(i)), so a + graduate loan, or one borrowed after every fitting child turned 24, is + never Parent PLUS. Every other code-3 loan is the head's + ``federal_parent_plus_loan_balance``: no listed adult child could have + borrowed it, so the student lives elsewhere and only the parent's loan is + reported, or it exceeds the dependent limit. An undergraduate loan within + the limit for a child of dependent age could be either, and is read as the + child's own, the household member the listing names; the receipt reports + the Parent PLUS share under the other reading too. - **IBR new-borrower status** is true when the person holds a federal loan and - every federal loan the person holds (own and Parent PLUS) was borrowed in - 2014 or later. The item records a year only, so 2014 counts as after July 1, - 2014. For a consolidated loan the year is that of the most recent underlying - loan. + every federal loan the person holds (own, and for the head Parent PLUS too, + since 34 CFR 685.209(b)(13)(ii) looks at any outstanding Direct or FFEL + balance) was borrowed in 2014 or later. The item records a year only, so + 2014 counts as after July 1, 2014. The share is an upper bound: the SCF sees + only loans still owed, so a borrower whose pre-2014 loans have been paid + off reads as new, and a consolidated loan is dated by its most recent + underlying loan, so a consolidation of pre-2014 loans can read as new too. - **Graduate share** is the share of the original amount of the person's own federal loans (Parent PLUS excluded) that financed a master's, professional or doctoral program (codes 13-15), weighted by the amount borrowed (falling @@ -62,11 +97,15 @@ reference person's age, sex and race, whether a spouse or partner is present, the number of children, wages, and income excluding capital gains. The first draws each recipient household's federal education debt from every SCF family. -The second, fit on borrower families only and conditioning also on that total, -draws the chained composition: the head's and spouse's shares of the total, -their graduate shares, and their new-borrower flags. Shares are clipped to -[0, 1] and rescaled when the two own shares exceed one; the remainder is the -head's Parent PLUS balance. +The second, fit on borrower families only, conditions also on that total and +on the number of adult children (children of the reference person or partner +aged 18 or over), and draws the chained composition: the Parent PLUS, adult +child's, head's and spouse's shares of the total, in that order, then the +three graduate shares and the three new-borrower flags. The four shares are +clipped to [0, 1] and normalized to sum to one, and a borrower drawn with no +share holds its own loan. Parent PLUS is therefore drawn for itself, first: a +family that resembles donors without Parent PLUS draws none, instead of +receiving whatever the own shares leave. Two of the shared SCF predictors are left out (:data:`SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS` keeps the full list for the @@ -88,22 +127,30 @@ a mixed-sex couple or else the older partner: the SCF's ordering of its economically dominant couple. ``is_married`` counts a cohabiting partner, as SCF ``MARRIED`` does, and children count those of the reference person or -partner, as SCF ``KIDS`` does. Incomes sum over the reference person, spouse or -partner and their children: the SCF primary economic unit, which leaves out a -financially independent co-resident relative or roommate. The spouse is the -member whose line number is the reference person's ``A_SPOUSE``, or else its -``PECOHAB`` partner when that column is present. A household drawn with a -spouse's balance but no spouse or partner present gives that balance to the -reference person, whose graduate share becomes the balance-weighted mean of -the two and whose new-borrower flag requires both. Everyone else gets zero. +partner, as SCF ``KIDS`` does; adult children are the ones aged 18 or over, +as listing code 4 aged 18 or over counts them. Incomes sum over the reference +person, spouse or partner and their children: the SCF primary economic unit, +which leaves out a financially independent co-resident relative or roommate. +The spouse is the member whose line number is the reference person's +``A_SPOUSE``, or else its ``PECOHAB`` partner when that column is present. A +household drawn with a spouse's balance but no spouse or partner present +gives that balance to the reference person, whose graduate share becomes the +balance-weighted mean of the two and whose new-borrower flag requires both. +The child's draw goes to the youngest child of the reference person or +partner aged 18 or over (``PEPAR1``/``PEPAR2``). A household without one +cannot use it, and the draw makes it the head's Parent PLUS loan, the SCF +side's rule for a loan whose student lives elsewhere. Everyone else gets zero. :data:`SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION` keeps the first receipt's construction (oldest adult, ``A_MARITL`` marriage, own children, whole household) for its before/after table. -**Known limits.** The SCF records the debts of the primary economic unit, so a -financially independent adult living in someone else's household is missed, -and an adult child's own loan in the unit is coded 3 and lands on the parent as -Parent PLUS. The SCF federal flag includes FFEL and Perkins loans that +**Known limits.** The SCF asks about the loans of the family living in the +household (its primary economic unit), so a financially independent adult in +someone else's household is missed unless the family reports the loan as one +for a child's education. No education-loan item names the listed person a +loan financed, and the public file bands listed ages, so the child rule works +from age bands, and with two or more adult children the youngest holds the +loan. The SCF federal flag includes FFEL and Perkins loans that ``federal_student_loan_balance`` excludes until consolidated. Balances are 2025 survey-date dollars. The FSA calibration (microcosm#1074) pins totals. """ @@ -135,12 +182,16 @@ __all__ = [ "SCF_2025_FULL_EXTRACT", "SCF_2025_SUMMARY_EXTRACT", + "SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR", "SCF_EDUCATION_DEBT_BALANCE_BANDS", "SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS", "SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS", - "SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS", "SCF_EDUCATION_DEBT_COMPOSITION_TARGETS", + "SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS", + "SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION", + "SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS", "SCF_EDUCATION_DEBT_PREDICTORS", + "SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION", "SCF_EDUCATION_DEBT_TOTAL_TARGET", "SCF_EDUCATION_DEBT_VINTAGE", "SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS", @@ -150,19 +201,22 @@ "SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN", "SCF_EDUCATION_LOAN_WHOSE_COLUMNS", "SCF_EDUCATION_LOAN_YEAR_COLUMNS", + "SCF_HOUSEHOLD_LISTING_AGE_COLUMNS", + "SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS", + "SCF_REVERSED_COLUMN", "SCF_WHOSE_EDUCATION_HOLDER", - "RecipientDefinition", - "SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION", - "SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION", - "ScfPublicFile", + "SCF_WHOSE_EDUCATION_UNREVERSED_LOANS", "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS", + "RecipientDefinition", + "ScfPublicFile", "derive_scf_education_debt_targets", "draw_scf_education_debt_households", "education_debt_fit_table", "fetch_scf_public_file", "impute_us_scf_education_debt", "load_scf_2025_education_debt_donor", + "scf_education_debt_composition_predictors", "scf_education_debt_donor_fit_table", "us_scf_education_debt_fit_table", "us_scf_education_debt_signal_gate", @@ -276,6 +330,64 @@ def zip_name(self) -> str: "x7951", ) _SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN = "x19" +#: "Original respondent and spouse/partner variables reversed": 1 = reversed. +SCF_REVERSED_COLUMN = "x8000" +_SCF_REVERSED = 1 +#: Loans (1-6) whose whose-education code the 2025 public file did not reverse +#: with the rest of a reversed record: loans 4 and 6 (X7893, X7993). See the +#: module docstring for the evidence. +SCF_WHOSE_EDUCATION_UNREVERSED_LOANS: tuple[int, ...] = (4, 6) +#: Household listing, persons 3-12: relationship to the respondent and age. +SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS: tuple[str, ...] = ( + "x108", + "x114", + "x120", + "x126", + "x132", + "x202", + "x208", + "x214", + "x220", + "x226", +) +SCF_HOUSEHOLD_LISTING_AGE_COLUMNS: tuple[str, ...] = ( + "x110", + "x116", + "x122", + "x128", + "x134", + "x204", + "x210", + "x216", + "x222", + "x228", +) +#: Listing relationship code 4: child (or child-in-law) of R or spouse/partner. +_SCF_LISTING_CHILD = 4 +#: The public file codes listed members' ages in bands (codebook X110): 1-3 as +#: 3, 4-6 as 6, 7-12 as 12, 13-17 as 17, 18-25 as 25, and older ages rounded +#: to the nearest 5 (so 26-27 also read 25), top-coded at 95. Code -> the +#: youngest and oldest age it can stand for. +_SCF_LISTING_AGE_BANDS: Mapping[int, tuple[float, float]] = { + 3: (0.0, 3.0), + 6: (4.0, 6.0), + 12: (7.0, 12.0), + 17: (13.0, 17.0), + 25: (18.0, 27.0), + 95: (93.0, 120.0), +} +#: The first listing age code that is certainly an adult (18-27). +_SCF_LISTING_FIRST_ADULT_CODE = 25 +#: The survey year that ages and years borrowed are counted from. +_SCF_SURVEY_YEAR = 2025 +#: Youngest age at which a loan can have financed a listed child's schooling. +_EARLIEST_STUDENT_AGE = 16.0 +#: A student 24 or older by December 31 of the award year is independent +#: (20 U.S.C. 1087vv(d)(1)(A)), so a parent cannot borrow Parent PLUS for them. +_INDEPENDENT_STUDENT_AGE = 24.0 +#: Aggregate limit on a dependent undergraduate's own Direct Subsidized and +#: Unsubsidized Loans (34 CFR 685.203(e)(1)). +_DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT = 31_000.0 _FEDERAL_LOAN_YES = 1 #: Programs counted as graduate or professional: master's (13), professional @@ -287,13 +399,16 @@ def zip_name(self) -> str: _HEAD = "head" _SPOUSE = "spouse" +_CHILD = "child" _PARENT_PLUS = "parent_plus" -#: Whose-education code -> who holds the loan in the build. See the module -#: docstring for the rule and its reasons. +_CHILD_EDUCATION = "child_education" +#: Whose-education code -> who holds the loan in the build. A loan for a +#: child's education is a co-resident child's own loan or the head's Parent +#: PLUS loan, by the rule in the module docstring. SCF_WHOSE_EDUCATION_HOLDER: Mapping[int, str] = { 1: _HEAD, 2: _SPOUSE, - 3: _PARENT_PLUS, + 3: _CHILD_EDUCATION, 6: _HEAD, 7: _HEAD, -7: _HEAD, @@ -301,14 +416,26 @@ def zip_name(self) -> str: #: Household-total target of the first QRF. SCF_EDUCATION_DEBT_TOTAL_TARGET = "federal_education_debt" -#: Chained targets of the second (borrower-family) QRF. -SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: tuple[str, ...] = ( +#: The second QRF's extra predictor: children of the reference person or +#: partner aged 18 or over in the household (SCF listing code 4). +SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR = "adult_children_in_household" +#: Each holder's share of the household total, drawn first in this order and +#: then normalized to sum to one. +SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS: tuple[str, ...] = ( + "parent_plus_share", + "child_own_share", "head_own_share", "spouse_own_share", +) +#: Chained targets of the second (borrower-family) QRF. +SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: tuple[str, ...] = ( + *SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS, "head_graduate_share", "spouse_graduate_share", + "child_graduate_share", "head_is_ibr_new_borrower", "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower", ) #: Every predictor both sides build: the shared SCF predictors plus income #: excluding capital gains. The diagnostics and ablations compare all of them. @@ -342,6 +469,19 @@ def zip_name(self) -> str: if predictor not in SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS ) + +def scf_education_debt_composition_predictors( + predictors: Sequence[str] = SCF_EDUCATION_DEBT_PREDICTORS, +) -> tuple[str, ...]: + """The second QRF's predictors: the first's, adult children and the total.""" + + return ( + *predictors, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + ) + + US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS: tuple[str, ...] = ( "federal_student_loan_balance", "federal_parent_plus_loan_balance", @@ -450,6 +590,9 @@ def zip_name(self) -> str: ) _OPTIONAL_PARTNER_COLUMN = "PECOHAB" +#: Age from which a CPS child of the reference person or partner is an adult. +_ADULT_AGE = 18.0 + #: Weighted share of households holding federal education debt allowed by the #: signal gate. The SCF 2025 share holding any education debt is 19.9%. _BORROWER_HOUSEHOLD_SHARE_BAND = (0.08, 0.35) @@ -563,22 +706,87 @@ def _numeric(table: pd.DataFrame, column: str) -> np.ndarray: ) +def _listing_age_range(code: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """The youngest and oldest age each public-file listing age code stands for. + + Codes 30-90 are ages rounded to the nearest 5, so ``code - 2`` to + ``code + 2``; zero or negative (no such person) gives NaN. + """ + + code = np.asarray(code, dtype=np.float64) + youngest = code - 2.0 + oldest = code + 2.0 + for band_code, (low, high) in _SCF_LISTING_AGE_BANDS.items(): + youngest = np.where(code == band_code, low, youngest) + oldest = np.where(code == band_code, high, oldest) + missing = code <= 0 + return np.where(missing, np.nan, youngest), np.where(missing, np.nan, oldest) + + +def _scf_whose_education_codes(full: pd.DataFrame) -> np.ndarray: + """Whose-education codes of loans 1-6, with the X8000 correction applied. + + In a reversed family (X8000 = 1) the public file swaps every respondent and + spouse/partner code so that code 1 is the reference person, except on + loans 4 and 6, whose codes still name the original respondent: those two + codes are swapped here. + """ + + whose = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_WHOSE_COLUMNS] + ).reshape(len(full), len(SCF_EDUCATION_LOAN_WHOSE_COLUMNS)) + reversed_family = _numeric(full, SCF_REVERSED_COLUMN) == _SCF_REVERSED + for loan in SCF_WHOSE_EDUCATION_UNREVERSED_LOANS: + column = whose[:, loan - 1] + whose[:, loan - 1] = np.where( + reversed_family & (column == 1), + 2.0, + np.where(reversed_family & (column == 2), 1.0, column), + ) + return whose + + +def _scf_adult_children(full: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]: + """Listed children aged 18 or over: a (rows x 10) mask and the age codes.""" + + shape = (len(full), len(SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS)) + relationship = np.column_stack( + [_numeric(full, c) for c in SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS] + ).reshape(shape) + age_code = np.column_stack( + [_numeric(full, c) for c in SCF_HOUSEHOLD_LISTING_AGE_COLUMNS] + ).reshape(shape) + adult_child = (relationship == _SCF_LISTING_CHILD) & ( + age_code >= _SCF_LISTING_FIRST_ADULT_CODE + ) + return adult_child, age_code + + def derive_scf_education_debt_targets(full: pd.DataFrame) -> pd.DataFrame: """Derive household federal education-debt targets from per-loan items. Pure and row-preserving: one output row per input (family, implicate) row. - See the module docstring for the federal, X7179, whose-education, IBR and - graduate-share rules this applies. + See the module docstring for the federal, X7179, X8000, whose-education, + co-resident child, IBR and graduate-share rules this applies. Returns columns: ``federal_education_debt``: federal balance after the X7179 rule. ``head_federal_student_loan_balance``, ``spouse_federal_student_loan_balance``, + ``child_federal_student_loan_balance``, ``federal_parent_plus_loan_balance``: its split by holder. - ``head_own_share``, ``spouse_own_share``: the two own balances over the - total (zero for a family without federal education debt). - ``head_is_ibr_new_borrower``, ``spouse_is_ibr_new_borrower``: 0/1. - ``head_graduate_share``, ``spouse_graduate_share``: in [0, 1]. + ``head_own_share``, ``spouse_own_share``, ``child_own_share``, + ``parent_plus_share``: each part over the total (zero for a family + without federal education debt). + ``head_is_ibr_new_borrower``, ``spouse_is_ibr_new_borrower``, + ``child_is_ibr_new_borrower``: 0/1. + ``head_graduate_share``, ``spouse_graduate_share``, + ``child_graduate_share``: in [0, 1]. + ``adult_children_in_household``: listed children aged 18 or over. + ``youngest_adult_child_age_code``: the youngest one's listing age code + (0 when there is none). + ``child_borrower_age_code``: the listing age code of the youngest + child a child-own loan fits (0 when there is none). ``education_debt``: every education loan, federal or not (EDN_INST without the education-coded installment loans). ``federal_status_unrecorded_balance``: loans 1-6 balances whose federal @@ -595,6 +803,9 @@ def derive_scf_education_debt_targets(full: pd.DataFrame) -> pd.DataFrame: *SCF_EDUCATION_LOAN_YEAR_COLUMNS, *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_REVERSED_COLUMN, + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, ) missing = [column for column in required if column not in full.columns] if missing: @@ -610,9 +821,7 @@ def derive_scf_education_debt_targets(full: pd.DataFrame) -> pd.DataFrame: federal_code = np.column_stack( [_numeric(full, c) for c in SCF_EDUCATION_LOAN_FEDERAL_COLUMNS] ).reshape(balances.shape) - whose = np.column_stack( - [_numeric(full, c) for c in SCF_EDUCATION_LOAN_WHOSE_COLUMNS] - ).reshape(balances.shape) + whose = _scf_whose_education_codes(full) year = np.column_stack( [_numeric(full, c) for c in SCF_EDUCATION_LOAN_YEAR_COLUMNS] ).reshape(balances.shape) @@ -639,14 +848,43 @@ def derive_scf_education_debt_targets(full: pd.DataFrame) -> pd.DataFrame: holder = np.full(balances.shape, _HEAD, dtype=object) for code, role in SCF_WHOSE_EDUCATION_HOLDER.items(): holder[whose == code] = role - is_head_own = federal & (holder == _HEAD) - is_spouse_own = federal & (holder == _SPOUSE) - is_parent_plus = federal & (holder == _PARENT_PLUS) + graduate = np.isin(program, list(_GRADUATE_PROGRAM_CODES)) - head_own = np.where(is_head_own, federal_balance, 0.0).sum(axis=1) - spouse_own = np.where(is_spouse_own, federal_balance, 0.0).sum(axis=1) - parent_plus = np.where(is_parent_plus, federal_balance, 0.0).sum(axis=1) - total = head_own + spouse_own + parent_plus + # A loan for a child's education is a co-resident child's own loan when a + # listed child aged 18 or over could have been in school when it was + # borrowed, unless only a parent could have borrowed it: an undergraduate + # amount above a dependent student's own aggregate limit, borrowed while + # the child could still have been dependent. Ages are the public file's + # bands; the arrays are (families x 10 listed persons x 6 loans). + adult_child, age_code = _scf_adult_children(full) + youngest_age, oldest_age = _listing_age_range(age_code) + years_ago = (_SCF_SURVEY_YEAR - year)[:, None, :] + # NaN ages (no such person) compare false, and adult_child excludes them. + fits = adult_child[:, :, None] & ( + oldest_age[:, :, None] - years_ago >= _EARLIEST_STUDENT_AGE + ) + could_be_dependent = ( + fits & (youngest_age[:, :, None] - years_ago < _INDEPENDENT_STUDENT_AGE) + ).any(axis=1) + parents_only = ( + could_be_dependent + & ~graduate + & (borrowed > _DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT) + ) + child_education = holder == _CHILD_EDUCATION + childs_own = child_education & fits.any(axis=1) & ~parents_only + holder[child_education] = _PARENT_PLUS + holder[childs_own] = _CHILD + + held = { + role: federal & (holder == role) + for role in (_HEAD, _SPOUSE, _CHILD, _PARENT_PLUS) + } + amount = { + role: np.where(mask, federal_balance, 0.0).sum(axis=1) + for role, mask in held.items() + } + total = amount[_HEAD] + amount[_SPOUSE] + amount[_CHILD] + amount[_PARENT_PLUS] def _share(numerator: np.ndarray) -> np.ndarray: return np.divide( @@ -664,7 +902,6 @@ def _new_borrower(holds: np.ndarray) -> np.ndarray: ) weight = np.where(borrowed > 0, borrowed, balances) - graduate = np.isin(program, list(_GRADUATE_PROGRAM_CODES)) def _graduate_share(holds: np.ndarray) -> np.ndarray: denominator = np.where(holds, weight, 0.0).sum(axis=1) @@ -676,17 +913,37 @@ def _graduate_share(holds: np.ndarray) -> np.ndarray: where=denominator > 0, ) + youngest_adult = np.where(adult_child, age_code, np.inf).min(axis=1) + fitted_codes = np.where(fits, age_code[:, :, None], np.inf).min(axis=1) + child_borrower = np.where(held[_CHILD], fitted_codes, np.inf).min(axis=1) + result = pd.DataFrame(index=full.index) result[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total - result["head_federal_student_loan_balance"] = head_own - result["spouse_federal_student_loan_balance"] = spouse_own - result["federal_parent_plus_loan_balance"] = parent_plus - result["head_own_share"] = _share(head_own) - result["spouse_own_share"] = _share(spouse_own) - result["head_graduate_share"] = _graduate_share(is_head_own) - result["spouse_graduate_share"] = _graduate_share(is_spouse_own) - result["head_is_ibr_new_borrower"] = _new_borrower(is_head_own | is_parent_plus) - result["spouse_is_ibr_new_borrower"] = _new_borrower(is_spouse_own) + result["head_federal_student_loan_balance"] = amount[_HEAD] + result["spouse_federal_student_loan_balance"] = amount[_SPOUSE] + result["child_federal_student_loan_balance"] = amount[_CHILD] + result["federal_parent_plus_loan_balance"] = amount[_PARENT_PLUS] + result["parent_plus_share"] = _share(amount[_PARENT_PLUS]) + result["child_own_share"] = _share(amount[_CHILD]) + result["head_own_share"] = _share(amount[_HEAD]) + result["spouse_own_share"] = _share(amount[_SPOUSE]) + result["head_graduate_share"] = _graduate_share(held[_HEAD]) + result["spouse_graduate_share"] = _graduate_share(held[_SPOUSE]) + result["child_graduate_share"] = _graduate_share(held[_CHILD]) + # The head's flag covers Parent PLUS loans too: 34 CFR 685.209(b)(13)(ii) + # looks at any outstanding Direct or FFEL balance. + result["head_is_ibr_new_borrower"] = _new_borrower(held[_HEAD] | held[_PARENT_PLUS]) + result["spouse_is_ibr_new_borrower"] = _new_borrower(held[_SPOUSE]) + result["child_is_ibr_new_borrower"] = _new_borrower(held[_CHILD]) + result[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = adult_child.sum( + axis=1 + ).astype(np.float64) + result["youngest_adult_child_age_code"] = np.where( + np.isfinite(youngest_adult), youngest_adult, 0.0 + ) + result["child_borrower_age_code"] = np.where( + np.isfinite(child_borrower), child_borrower, 0.0 + ) result["education_debt"] = loans_1_6 + remaining result["federal_status_unrecorded_balance"] = np.where( has_balance & (federal_code == 0), balances, 0.0 @@ -708,9 +965,10 @@ def load_scf_2025_education_debt_donor( check, for test fixtures), and the per-loan balances must reproduce the summary extract's ``edn_inst``. Returns one row per (family, implicate) with a positive weight: the predictors, the targets of - :func:`derive_scf_education_debt_targets`, the ``scf_weight`` donor weight - and the fit-table covariates (``racecl4``, ``income``, ``educ``, - ``spouse_age``, ``yy1``, ``implicate``). + :func:`derive_scf_education_debt_targets` (which include the second QRF's + ``adult_children_in_household``), the ``scf_weight`` donor weight and the + fit-table covariates (``racecl4``, ``income``, ``educ``, ``spouse_age``, + ``yy1``, ``implicate``). """ summary_columns = ( @@ -740,6 +998,9 @@ def load_scf_2025_education_debt_donor( *SCF_EDUCATION_LOAN_YEAR_COLUMNS, *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_REVERSED_COLUMN, + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, ) summary = _read_stata_columns( summary_extract_path, summary_columns, label="2025 summary extract" @@ -867,11 +1128,12 @@ def __post_init__(self) -> None: @dataclass(frozen=True) class _Recipients: - """Household-ordered predictors and each household's head and spouse rows.""" + """Household-ordered predictors and each household's recipient rows.""" features: pd.DataFrame head_position: np.ndarray spouse_position: np.ndarray # -1 where no spouse or partner is present + child_position: np.ndarray # the youngest adult child; -1 where none def _line_lookup(person: pd.DataFrame) -> dict[tuple[object, int], int]: @@ -983,6 +1245,52 @@ def _children_counts( return counts +def _adult_children( + person: pd.DataFrame, + head_position: np.ndarray, + spouse_position: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Children of the head or partner aged 18 or over, and the youngest one. + + A child is a member whose CPS parent pointer (``PEPAR1``/``PEPAR2``) names + the head or the spouse or partner, as SCF listing code 4 counts children of + the respondent or spouse/partner. Returns the count per household and the + positional row of the youngest (lowest line number on a tie), or -1. + """ + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + age = pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + parent_of: dict[tuple[object, int], int] = { + (household[head], int(line[head])): index + for index, head in enumerate(head_position) + } + for index, spouse in enumerate(spouse_position): + if spouse >= 0: + parent_of.setdefault((household[spouse], int(line[spouse])), index) + pointers = [ + pd.to_numeric(person[column], errors="coerce").fillna(-1).to_numpy() + for column in ("PEPAR1", "PEPAR2") + ] + counts = np.zeros(len(head_position), dtype=np.float64) + youngest = np.full(len(head_position), -1, dtype=np.int64) + for row in np.flatnonzero(age >= _ADULT_AGE): + owners = { + parent_of.get((household[row], int(pointer[row]))) + for pointer in pointers + if pointer[row] > 0 + } + owners.discard(None) + if not owners: + continue + owner = next(iter(owners)) + counts[owner] += 1.0 + current = youngest[owner] + if current < 0 or (age[row], line[row]) < (age[current], line[current]): + youngest[owner] = row + return counts, youngest + + def _income_unit_owner( person: pd.DataFrame, head_position: np.ndarray, @@ -1117,11 +1425,22 @@ def _recipients( sums = np.zeros(len(head_position), dtype=np.float64) np.add.at(sums, owner[member], values[member]) features[column] = sums + adult_children, child_position = _adult_children( + person, head_position, spouse_position + ) + features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = adult_children features.index = household.index return _Recipients( - features=features.loc[:, list(SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS)], + features=features.loc[ + :, + [ + *SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + ], + ], head_position=head_position, spouse_position=spouse_position, + child_position=child_position, ) @@ -1149,6 +1468,20 @@ def _fit_frame(donor: pd.DataFrame, columns: Sequence[str]) -> pd.DataFrame: return fit +def _split_whole_dollars(total: np.ndarray, shares: np.ndarray) -> np.ndarray: + """Split whole-dollar totals by row shares; a zero share stays zero. + + Each part is the floor of its share of the total, and the few dollars of + remainder go to the row's largest share, so the parts sum to the total. + """ + + parts = np.floor(total[:, None] * shares) + remainder = total - parts.sum(axis=1) + largest = np.argmax(shares, axis=1) + parts[np.arange(len(total)), largest] += remainder + return parts + + def draw_scf_education_debt_households( features: pd.DataFrame, donor: pd.DataFrame, @@ -1159,11 +1492,20 @@ def draw_scf_education_debt_households( ) -> pd.DataFrame: """Draw household totals and their composition for recipient rows. - ``features`` carries the predictor columns, one row per recipient - household (or held-out SCF family). Returns, on the same index, the drawn - total and the normalized composition: the head's and spouse's own - balances, the head's Parent PLUS balance, both graduate shares and both - new-borrower flags. Used by the build and by the SCF hold-out fit check. + ``features`` carries the predictor columns and + ``adult_children_in_household``, one row per recipient household (or + held-out SCF family). Returns, on the same index, the drawn total and the + normalized composition: the head's, spouse's and adult child's own + balances, the head's Parent PLUS balance, the three graduate shares and the + three new-borrower flags. Used by the build and by the SCF hold-out fit + check. + + The four holder shares are drawn in turn (Parent PLUS first, so a family + unlike any Parent PLUS donor draws none) and normalized to sum to one; a + borrower whose every share is drawn zero holds its own loan. A child's + share drawn for a household without an adult child is the head's Parent + PLUS loan: as on the SCF side, a loan for a child who is not in the + household is the parent's. """ from microcosm.fit import QRF @@ -1195,7 +1537,9 @@ def draw_scf_education_debt_households( result[column] = 0.0 borrowers = drawn_total > 0 if borrowers.any(): - composition_predictors = [*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET] + composition_predictors = list( + scf_education_debt_composition_predictors(predictors) + ) borrower_donor = donor.loc[ pd.to_numeric(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], errors="coerce") > 0 ] @@ -1215,39 +1559,66 @@ def draw_scf_education_debt_households( targets=list(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS), weights=_DONOR_WEIGHT_COLUMN, ) - recipients = features.loc[borrowers, predictors].copy() + recipients = features.loc[borrowers, composition_predictors[:-1]].copy() recipients[SCF_EDUCATION_DEBT_TOTAL_TARGET] = drawn_total[borrowers] drawn = composition_model.predict(recipients, release_models=True) for column in SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: result.loc[borrowers, column] = np.asarray(drawn[column], dtype=np.float64) - head_share = result["head_own_share"].clip(0.0, 1.0).to_numpy() - spouse_share = result["spouse_own_share"].clip(0.0, 1.0).to_numpy() - own = head_share + spouse_share - scale = np.where(own > 1.0, 1.0 / np.where(own > 0, own, 1.0), 1.0) - head_share = head_share * scale - spouse_share = spouse_share * scale - head_own = np.round(drawn_total * head_share) - # Whole dollars that never exceed the drawn total once rounded. - spouse_own = np.minimum( - np.round(drawn_total * spouse_share), drawn_total - head_own + shares = np.column_stack( + [ + result[column].clip(0.0, 1.0).to_numpy(dtype=np.float64) + for column in SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS + ] ) - parent_plus = np.maximum(drawn_total - head_own - spouse_own, 0.0) - result["head_federal_student_loan_balance"] = head_own - result["spouse_federal_student_loan_balance"] = spouse_own - result["federal_parent_plus_loan_balance"] = parent_plus - for prefix, own_balance in (("head", head_own), ("spouse", spouse_own)): + plus, child, head, spouse = range(len(SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS)) + shares[~borrowers] = 0.0 + shares[borrowers & (shares.sum(axis=1) <= 0), head] = 1.0 + no_adult_child = ( + pd.to_numeric( + features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR], errors="coerce" + ) + .fillna(0.0) + .to_numpy() + <= 0 + ) + child_new = result["child_is_ibr_new_borrower"].to_numpy() >= 0.5 + head_new = result["head_is_ibr_new_borrower"].to_numpy() >= 0.5 + moved = no_adult_child & (shares[:, child] > 0) + # The head then holds Parent PLUS for that child too: new only when every + # loan the head holds is. + head_held = (shares[:, head] + shares[:, plus]) > 0 + head_new = np.where( + moved, np.where(head_held, head_new, True) & child_new, head_new + ) + shares[moved, plus] += shares[moved, child] + shares[moved, child] = 0.0 + sums = shares.sum(axis=1) + shares = np.divide( + shares, sums[:, None], out=np.zeros_like(shares), where=sums[:, None] > 0 + ) + parts = _split_whole_dollars(drawn_total, shares) + result["head_federal_student_loan_balance"] = parts[:, head] + result["spouse_federal_student_loan_balance"] = parts[:, spouse] + result["child_federal_student_loan_balance"] = parts[:, child] + result["federal_parent_plus_loan_balance"] = parts[:, plus] + for prefix, column in (("head", head), ("spouse", spouse), ("child", child)): graduate = result[f"{prefix}_graduate_share"].clip(0.0, 1.0).to_numpy() - result[f"{prefix}_graduate_share"] = np.where(own_balance > 0, graduate, 0.0) - head_holds = (head_own + parent_plus) > 0 + result[f"{prefix}_graduate_share"] = np.where( + parts[:, column] > 0, graduate, 0.0 + ) result["head_is_ibr_new_borrower"] = ( - head_holds & (result["head_is_ibr_new_borrower"].to_numpy() >= 0.5) + ((parts[:, head] + parts[:, plus]) > 0) & head_new ).astype(np.float64) result["spouse_is_ibr_new_borrower"] = ( - (spouse_own > 0) & (result["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5) + (parts[:, spouse] > 0) + & (result["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5) ).astype(np.float64) - result["head_own_share"] = head_share - result["spouse_own_share"] = spouse_share + result["child_is_ibr_new_borrower"] = ((parts[:, child] > 0) & child_new).astype( + np.float64 + ) + for index, column in enumerate(SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS): + result[column] = shares[:, index] return result @@ -1263,11 +1634,11 @@ def impute_us_scf_education_debt( """Impute the four person columns onto a US frame. Returns a DataFrame indexed like the person table with - :data:`US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS`; only reference persons and - their spouses or partners can be nonzero. ``definition`` selects the CPS - construction of the SCF predictors and ``predictors`` the subset the QRFs - condition on; the receipts pass the first receipt's choices for their - before/after table. + :data:`US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS`; only reference persons, + their spouses or partners, and the youngest child of either aged 18 or + over can be nonzero. ``definition`` selects the CPS construction of the + SCF predictors and ``predictors`` the subset the QRFs condition on; the + receipts pass the first receipt's choices for their before/after table. """ recipients = _recipients(frame, definition) @@ -1287,13 +1658,26 @@ def impute_us_scf_education_debt( head = recipients.head_position spouse = recipients.spouse_position + child = recipients.child_position has_spouse = spouse >= 0 + has_child = child >= 0 head_own = drawn["head_federal_student_loan_balance"].to_numpy() spouse_own = drawn["spouse_federal_student_loan_balance"].to_numpy() + child_own = drawn["child_federal_student_loan_balance"].to_numpy() + head_plus = drawn["federal_parent_plus_loan_balance"].to_numpy() head_graduate = drawn["head_graduate_share"].to_numpy() spouse_graduate = drawn["spouse_graduate_share"].to_numpy() + child_graduate = drawn["child_graduate_share"].to_numpy() head_new = drawn["head_is_ibr_new_borrower"].to_numpy() >= 0.5 spouse_new = drawn["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5 + child_new = drawn["child_is_ibr_new_borrower"].to_numpy() >= 0.5 + # The draw gives a child's balance only to households with an adult child, + # which is exactly where the youngest one was found. + if ((child_own > 0) & ~has_child).any(): + raise ValueError( + "SCF education-debt draw gave a child's balance to a household " + "without an adult child." + ) # Without a spouse or partner present, the reference person holds the # spouse's draw too: a balance-weighted graduate share, and new-borrower @@ -1310,22 +1694,24 @@ def impute_us_scf_education_debt( ), head_graduate, ) - head_holds_any = ( - head_own + drawn["federal_parent_plus_loan_balance"].to_numpy() - ) > 0 + head_holds_any = (head_own + head_plus) > 0 combined_new = np.where( moved, np.where(head_holds_any, head_new, True) & spouse_new, head_new, ) own[head] = combined_own - parent_plus[head] = drawn["federal_parent_plus_loan_balance"].to_numpy() + parent_plus[head] = head_plus graduate[head] = combined_graduate new_borrower[head] = combined_new spouse_rows = spouse[has_spouse] own[spouse_rows] = spouse_own[has_spouse] graduate[spouse_rows] = spouse_graduate[has_spouse] new_borrower[spouse_rows] = spouse_new[has_spouse] + child_rows = child[has_child] + own[child_rows] = child_own[has_child] + graduate[child_rows] = child_graduate[has_child] + new_borrower[child_rows] = child_new[has_child] holds_any = (own + parent_plus) > 0 result = pd.DataFrame(index=person.index) @@ -1433,7 +1819,7 @@ def us_scf_education_debt_stage_spec() -> SourceStageSpec: (SCF_EDUCATION_DEBT_TOTAL_TARGET,), ), "scf_2025_education_debt_borrowers": ( - (*SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_TOTAL_TARGET), + scf_education_debt_composition_predictors(), SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, ), } @@ -1620,6 +2006,33 @@ def _group(mask: np.ndarray) -> dict[str, float]: } +def _spread_listing_ages( + codes: np.ndarray, weights: np.ndarray, balances: np.ndarray +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Spread each listing age code's weight evenly over the ages it stands for. + + The public file bands listed members' ages (18-27 read 25), so a child + borrower's weight is split evenly across the whole years of the band for + the borrower age table. Returns ages, weights and balances, one row per + (borrower, year of age). + """ + + youngest, oldest = _listing_age_range(codes) + ages: list[np.ndarray] = [] + spread: list[np.ndarray] = [] + values: list[np.ndarray] = [] + for low, high, weight, balance in zip( + youngest, oldest, weights, balances, strict=True + ): + years = np.arange(low, min(high, 99.0) + 1.0) + ages.append(years) + spread.append(np.full(len(years), weight / len(years))) + values.append(np.full(len(years), balance)) + if not ages: + return np.zeros(0), np.zeros(0), np.zeros(0) + return np.concatenate(ages), np.concatenate(spread), np.concatenate(values) + + def scf_education_debt_donor_fit_table( donor: pd.DataFrame, *, @@ -1631,45 +2044,72 @@ def scf_education_debt_donor_fit_table( donor's index) replaces the observed targets, so a hold-out draw and the observed SCF are tabulated identically. Implicates are pooled with ``wgt`` (each implicate carries one fifth of the population), as the Fed - pools them for medians. + pools them for medians. Borrowers are the reference person, the spouse or + partner and the adult child holding a child-own balance, whose age is the + child's listing age band (the child the loan fits when observed, the + youngest adult child when drawn), spread evenly over the band. """ source = donor if imputed is None else imputed weight = donor[_DONOR_WEIGHT_COLUMN].to_numpy(dtype=np.float64) - head_own = source["head_federal_student_loan_balance"].to_numpy(dtype=np.float64) - spouse_own = source["spouse_federal_student_loan_balance"].to_numpy( - dtype=np.float64 - ) - parent_plus = source["federal_parent_plus_loan_balance"].to_numpy(dtype=np.float64) + + def _column(name: str) -> np.ndarray: + return source[name].to_numpy(dtype=np.float64) + + head_own = _column("head_federal_student_loan_balance") + spouse_own = _column("spouse_federal_student_loan_balance") + child_own = _column("child_federal_student_loan_balance") + parent_plus = _column("federal_parent_plus_loan_balance") head_balance = head_own + parent_plus head_holds = head_balance > 0 spouse_holds = spouse_own > 0 + child_holds = child_own > 0 head_age = donor["age"].to_numpy(dtype=np.float64) spouse_age = donor["spouse_age"].to_numpy(dtype=np.float64) - head_graduate = source["head_graduate_share"].to_numpy(dtype=np.float64) - spouse_graduate = source["spouse_graduate_share"].to_numpy(dtype=np.float64) - head_new = source["head_is_ibr_new_borrower"].to_numpy(dtype=np.float64) >= 0.5 - spouse_new = source["spouse_is_ibr_new_borrower"].to_numpy(dtype=np.float64) >= 0.5 + child_age_code = donor[ + "child_borrower_age_code" + if imputed is None + else "youngest_adult_child_age_code" + ].to_numpy(dtype=np.float64) + child_ages, child_weights, child_balances = _spread_listing_ages( + child_age_code[child_holds], weight[child_holds], child_own[child_holds] + ) + head_graduate = _column("head_graduate_share") + spouse_graduate = _column("spouse_graduate_share") + child_graduate = _column("child_graduate_share") + head_new = _column("head_is_ibr_new_borrower") >= 0.5 + spouse_new = _column("spouse_is_ibr_new_borrower") >= 0.5 + child_new = _column("child_is_ibr_new_borrower") >= 0.5 income = donor["household_income"].to_numpy(dtype=np.float64) return education_debt_fit_table( - household_debt=head_own + spouse_own + parent_plus, + household_debt=head_own + spouse_own + child_own + parent_plus, household_weight=weight, head_age=head_age, race_group=donor["racecl4"].to_numpy(), income_quintile=_income_quintiles(income, weight), borrower_balance=np.concatenate( - [head_balance[head_holds], spouse_own[spouse_holds]] + [head_balance[head_holds], spouse_own[spouse_holds], child_balances] + ), + borrower_age=np.concatenate( + [head_age[head_holds], spouse_age[spouse_holds], child_ages] + ), + borrower_weight=np.concatenate( + [weight[head_holds], weight[spouse_holds], child_weights] ), - borrower_age=np.concatenate([head_age[head_holds], spouse_age[spouse_holds]]), - borrower_weight=np.concatenate([weight[head_holds], weight[spouse_holds]]), parent_plus_total=float(np.dot(parent_plus, weight)), graduate_numerator=float( - np.dot(head_own * head_graduate + spouse_own * spouse_graduate, weight) + np.dot( + head_own * head_graduate + + spouse_own * spouse_graduate + + child_own * child_graduate, + weight, + ) ), - graduate_denominator=float(np.dot(head_own + spouse_own, weight)), + graduate_denominator=float(np.dot(head_own + spouse_own + child_own, weight)), new_borrower_weight=float( weight[head_holds & head_new].sum() + weight[spouse_holds & spouse_new].sum() + + weight[child_holds & child_new].sum() ), ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index f0891cedf4..5d3309e907 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597", + "sha256": "f97bbca65e9ceee16311f2879e2729c884c96af3c9d7a02b531b5b97a57e309a", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py index 9e6dc5c416..d276c72f9a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py @@ -20,7 +20,9 @@ from microcosm.build.us_runtime.scf_education_debt import ( SCF_2025_FULL_EXTRACT, SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS, SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_TOTAL_TARGET, SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, @@ -30,6 +32,8 @@ SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, SCF_EDUCATION_LOAN_WHOSE_COLUMNS, SCF_EDUCATION_LOAN_YEAR_COLUMNS, + SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, ScfPublicFile, derive_scf_education_debt_targets, @@ -38,6 +42,7 @@ fetch_scf_public_file, impute_us_scf_education_debt, load_scf_2025_education_debt_donor, + scf_education_debt_composition_predictors, scf_education_debt_donor_fit_table, us_scf_education_debt_fit_table, us_scf_education_debt_signal_gate, @@ -60,11 +65,29 @@ } -def _family(*loans: dict[str, float], remaining: float = 0.0) -> dict[str, float]: - """One full-file row: up to six loans given as item dictionaries.""" +def _family( + *loans: dict[str, float], + remaining: float = 0.0, + reversed_codes: bool = False, + listed: tuple[tuple[int, int], ...] = (), +) -> dict[str, float]: + """One full-file row: up to six loans given as item dictionaries. + + ``listed`` gives household-listing persons 3-12 as (relationship, public + age code) pairs; ``reversed_codes`` sets X8000 = 1. + """ row = {column: 0.0 for columns in _LOAN_ITEMS.values() for column in columns} row[SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN] = remaining + row["x8000"] = 1.0 if reversed_codes else 5.0 + for column in ( + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + ): + row[column] = 0.0 + for index, (relationship, age_code) in enumerate(listed): + row[SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS[index]] = float(relationship) + row[SCF_HOUSEHOLD_LISTING_AGE_COLUMNS[index]] = float(age_code) for index, loan in enumerate(loans): for item, value in loan.items(): row[_LOAN_ITEMS[item][index]] = value @@ -196,6 +219,111 @@ def test_derivation_refuses_a_missing_loan_item() -> None: frame = pd.DataFrame.from_records([_family(_loan(1_000))]).drop(columns="x9338") with pytest.raises(ValueError, match="x9338"): derive_scf_education_debt_targets(frame) + listing = pd.DataFrame.from_records([_family(_loan(1_000))]).drop(columns="x8000") + with pytest.raises(ValueError, match="x8000"): + derive_scf_education_debt_targets(listing) + + +def test_reversed_families_swap_the_codes_of_loans_4_and_6_back() -> None: + loans = [_loan(1_000.0 * (index + 1), whose=1) for index in range(6)] + reversed_row, unreversed_row = _derive( + _family(*loans, reversed_codes=True), _family(*loans) + ).to_dict("records") + + # Loans 4 and 6 (4,000 and 6,000) still name the original respondent, now + # the spouse or partner, in a reversed family; loans 1, 2, 3 and 5 do not. + assert reversed_row["head_federal_student_loan_balance"] == 11_000 + assert reversed_row["spouse_federal_student_loan_balance"] == 10_000 + assert unreversed_row["head_federal_student_loan_balance"] == 21_000 + assert unreversed_row["spouse_federal_student_loan_balance"] == 0.0 + spouse_coded = _derive( + _family( + _loan(1_000, whose=2), + _loan(1_000, whose=2), + _loan(1_000, whose=2), + _loan(4_000, whose=2), + _loan(1_000, whose=3), + _loan(6_000, whose=2), + reversed_codes=True, + ) + ).iloc[0] + assert spouse_coded["head_federal_student_loan_balance"] == 10_000 + assert spouse_coded["spouse_federal_student_loan_balance"] == 3_000 + # Code 3 names neither partner, so it is never swapped. + assert spouse_coded["federal_parent_plus_loan_balance"] == 1_000 + + +_CHILD = 4 +_GRANDCHILD = 5 + + +@pytest.mark.parametrize( + ("listed", "loan", "holder"), + [ + # A child aged 18-27 (code 25) who could have borrowed it in 2022. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2022), "child"), + # No listed child: the student lives elsewhere. + ((), _loan(9_000, whose=3, year=2022), "parent_plus"), + # Only a 13-17-year-old (code 17) and a grandchild. + (((_CHILD, 17), (_GRANDCHILD, 25)), _loan(9_000, whose=3, year=2022), + "parent_plus"), + # At most 27 now, so at most 2 when a 2000 loan was borrowed. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2000), "parent_plus"), + # Above the $31,000 dependent limit while possibly dependent. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2022, borrowed=40_000), + "parent_plus"), + # At the limit itself, which the child's own loans can reach. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2022, borrowed=31_000), + "child"), + # A graduate loan is never Parent PLUS, whatever its size. + (((_CHILD, 25),), + _loan(9_000, whose=3, year=2022, program=13, borrowed=80_000), "child"), + # A child aged 38-42 (code 40) was at least 28 in 2015: independent. + (((_CHILD, 40),), _loan(9_000, whose=3, year=2015, borrowed=50_000), + "child"), + # Aged 28-32 (code 30), so possibly 23 in 2018: possibly dependent. + (((_CHILD, 30),), _loan(9_000, whose=3, year=2018, borrowed=50_000), + "parent_plus"), + ], +) # fmt: skip +def test_loans_for_a_child_go_to_a_co_resident_adult_child_who_fits( + listed, loan, holder +) -> None: + row = _derive(_family(loan, listed=listed)).iloc[0] + + child = row["child_federal_student_loan_balance"] + parent_plus = row["federal_parent_plus_loan_balance"] + expected = (9_000, 0.0) if holder == "child" else (0.0, 9_000) + assert (child, parent_plus) == expected + assert row["child_own_share"] + row["parent_plus_share"] == 1.0 + assert row["head_federal_student_loan_balance"] == 0.0 + + +def test_child_targets_carry_the_childs_flags_shares_and_ages() -> None: + row = _derive( + _family( + _loan(10_000, whose=1, year=2010), + _loan(6_000, whose=3, year=2020, program=12), + _loan(4_000, whose=3, year=2023, program=13, borrowed=12_000), + _loan(2_000, whose=3, year=2001), + listed=((_CHILD, 30), (_CHILD, 25), (_CHILD, 12)), + ) + ).iloc[0] + + assert row["child_federal_student_loan_balance"] == 10_000 + # The 2001 loan fits no listed adult child (at most 32 now, 8 then). + assert row["federal_parent_plus_loan_balance"] == 2_000 + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 22_000 + assert row["child_own_share"] == pytest.approx(10_000 / 22_000) + assert row["child_graduate_share"] == pytest.approx(12_000 / 18_000) + assert row["child_is_ibr_new_borrower"] == 1.0 + # The head holds a 2010 loan and the 2001 Parent PLUS loan. + assert row["head_is_ibr_new_borrower"] == 0.0 + assert row[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] == 2.0 + assert row["youngest_adult_child_age_code"] == 25.0 + assert row["child_borrower_age_code"] == 25.0 + shares = [f"{holder}_own_share" for holder in ("head", "spouse", "child")] + assert row[[*shares, "parent_plus_share"]].sum() == pytest.approx(1.0) # --------------------------------------------------------------------------- # @@ -304,7 +432,7 @@ def test_stage_declares_the_person_outputs_and_pinned_2025_files() -> None: assert spec.artifacts[1]["vintage"] == "2022" assert spec.artifacts[2]["member"] == "pu2023.csv" # Both QRFs condition on the production predictors; the borrower fit adds - # the drawn total. + # the adult children and the drawn total. fits = { operation.parameters["donor"]: operation.parameters for operation in spec.operations @@ -316,8 +444,17 @@ def test_stage_declares_the_person_outputs_and_pinned_2025_files() -> None: ) assert tuple(fits["scf_2025_education_debt_borrowers"]["predictors"]) == ( *SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, SCF_EDUCATION_DEBT_TOTAL_TARGET, ) + assert scf_education_debt_composition_predictors() == tuple( + fits["scf_2025_education_debt_borrowers"]["predictors"] + ) + assert tuple(fits["scf_2025_education_debt_borrowers"]["targets"]) == ( + SCF_EDUCATION_DEBT_COMPOSITION_TARGETS + ) + # Parent PLUS is drawn first, for itself, not left as a residual. + assert SCF_EDUCATION_DEBT_COMPOSITION_TARGETS[0] == "parent_plus_share" def test_production_predictors_drop_only_the_reviewed_exclusions() -> None: @@ -406,32 +543,56 @@ def _ready_donor(n: int = 600) -> pd.DataFrame: donor["interest_dividend_income"] = rng.gamma(1.0, 800.0, n) donor["social_security_pension_income"] = rng.gamma(1.0, 5_000.0, n) donor["household_income"] = donor["employment_income"] + rng.gamma(1.0, 5_000.0, n) + donor[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = np.where( + donor["age"] >= 40, rng.integers(0, 3, n), 0 + ).astype(np.float64) borrower = (rng.random(n) < 0.2) & (donor["age"] < 65) total = np.where(borrower, rng.gamma(1.5, 25_000.0, n).round(), 0.0) - holder = rng.choice(3, n, p=[0.6, 0.25, 0.15]) - head_share = np.where(borrower, (holder == 0).astype(float), 0.0) - spouse_share = np.where(borrower, (holder == 1).astype(float), 0.0) - donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total - donor["head_own_share"] = head_share - donor["spouse_own_share"] = spouse_share - donor["head_graduate_share"] = np.where( - head_share > 0, rng.choice([0.0, 0.5, 1.0], n), 0.0 - ) - donor["spouse_graduate_share"] = np.where( - spouse_share > 0, rng.choice([0.0, 1.0], n), 0.0 - ) - donor["head_is_ibr_new_borrower"] = np.where( - borrower & (holder != 1), (rng.random(n) < 0.5).astype(float), 0.0 - ) - donor["spouse_is_ibr_new_borrower"] = np.where( - spouse_share > 0, (rng.random(n) < 0.5).astype(float), 0.0 + # Holders: 0 head, 1 spouse, 2 Parent PLUS, 3 an adult child. + holder = rng.choice(4, n, p=[0.55, 0.25, 0.1, 0.1]) + holder = np.where( + (holder == 3) & (donor[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] == 0), + 0, + holder, ) + for index, column in enumerate( + ("head_own_share", "spouse_own_share", "parent_plus_share", "child_own_share") + ): + donor[column] = np.where(borrower, (holder == index).astype(float), 0.0) + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total + for prefix, choices in ( + ("head", [0.0, 0.5, 1.0]), + ("spouse", [0.0, 1.0]), + ("child", [0.0, 1.0]), + ): + donor[f"{prefix}_graduate_share"] = np.where( + donor[f"{prefix}_own_share"] > 0, rng.choice(choices, n), 0.0 + ) + head_holds = (donor["head_own_share"] + donor["parent_plus_share"]) > 0 + for prefix, holds in ( + ("head", head_holds), + ("spouse", donor["spouse_own_share"] > 0), + ("child", donor["child_own_share"] > 0), + ): + donor[f"{prefix}_is_ibr_new_borrower"] = np.where( + holds, (rng.random(n) < 0.5).astype(float), 0.0 + ) donor[_DONOR_WEIGHT_COLUMN] = rng.uniform(500.0, 2_000.0, n) return donor -def _person_rows(n_households: int = 150) -> pd.DataFrame: - """Couples (A_SPOUSE pointers) and single adults with a child.""" +def _features(donor: pd.DataFrame) -> pd.DataFrame: + return donor.loc[ + :, [*SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] + ] + + +def _person_rows(n_households: int = 150, *, adult_children: bool = False): + """Couples (A_SPOUSE pointers) and single adults with a child. + + With ``adult_children``, every couple's child is 19 to 24 and every fourth + couple also has a 30-year-old child (line 4). + """ rng = np.random.default_rng(97) records: list[dict[str, object]] = [] @@ -443,8 +604,16 @@ def _person_rows(n_households: int = 150) -> pd.DataFrame: if couple else [(1, 0, 7, 0), (2, 0, 7, 1)] ) + if adult_children and couple and household_id % 4 == 0: + members.append((4, 0, 7, 1)) for line, spouse, marital, female in members: - child = line == 3 or (not couple and line == 2) + child = line >= 3 or (not couple and line == 2) + if not child: + age = float(rng.integers(24, 60)) + elif adult_children and couple: + age = 30.0 if line == 4 else float(rng.integers(19, 25)) + else: + age = float(rng.integers(5, 17)) records.append( { "person_id": person_id, @@ -452,9 +621,7 @@ def _person_rows(n_households: int = 150) -> pd.DataFrame: "PH_SEQ": household_id, "A_LINENO": line, "A_SPOUSE": spouse, - "age": float( - rng.integers(5, 17) if child else rng.integers(24, 60) - ), + "age": age, "is_female": bool(female), "PRDTRACE": int(rng.integers(1, 5)), "PRDTHSP": int(rng.integers(0, 2)), @@ -501,7 +668,7 @@ def _us_frame(person: pd.DataFrame) -> Frame: def test_household_draws_are_seeded_coherent_and_nonnegative() -> None: donor = _ready_donor() - features = donor.loc[:, list(SCF_EDUCATION_DEBT_PREDICTORS)].iloc[:200] + features = _features(donor).iloc[:200] first = draw_scf_education_debt_households(features, donor, seed=3, n_estimators=15) second = draw_scf_education_debt_households( features, donor, seed=3, n_estimators=15 @@ -512,18 +679,59 @@ def test_household_draws_are_seeded_coherent_and_nonnegative() -> None: parts = ( first["head_federal_student_loan_balance"] + first["spouse_federal_student_loan_balance"] + + first["child_federal_student_loan_balance"] + first["federal_parent_plus_loan_balance"] ).to_numpy() assert (total >= 0).all() and (total > 0).any() np.testing.assert_array_equal(parts, total) - assert (first["head_own_share"] + first["spouse_own_share"] <= 1 + 1e-9).all() borrowers = total > 0 + shares = first[list(SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS)].sum(axis=1) + np.testing.assert_allclose(shares[borrowers], 1.0) assert not first.loc[~borrowers, list(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS)].any( axis=None ) + # A household without an adult child holds no child balance. + no_adult_child = features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] == 0 + assert (first.loc[no_adult_child, "child_federal_student_loan_balance"] == 0).all() + + +def test_a_donor_family_without_parent_plus_rarely_draws_one() -> None: + """Parent PLUS is drawn for itself, not left over by the own shares. + + Own debt splits between the two partners in fractions, and only families + with a reference person 55 or older hold Parent PLUS. The draw for every + donor family's own predictors gives Parent PLUS to few of the others. + """ + + rng = np.random.default_rng(1074) + donor = _ready_donor(1_500) + borrower = donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0 + older = donor["age"] >= 55 + head_share = rng.uniform(0.2, 0.8, len(donor)) + donor["head_own_share"] = np.where(borrower & ~older, head_share, 0.0) + donor["spouse_own_share"] = np.where(borrower & ~older, 1.0 - head_share, 0.0) + donor["parent_plus_share"] = np.where(borrower & older, 1.0, 0.0) + donor["child_own_share"] = 0.0 + donor["child_graduate_share"] = 0.0 + donor["child_is_ibr_new_borrower"] = 0.0 + drawn = draw_scf_education_debt_households( + _features(donor), donor, seed=8, n_estimators=20 + ) + with_debt = drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0 + draws_plus = drawn["federal_parent_plus_loan_balance"] > 0 + assert draws_plus[with_debt & ~older].mean() < 0.02 + assert draws_plus[with_debt & older].mean() > 0.9 + # A donor with no Parent PLUS at all never draws it. + donor["parent_plus_share"] = 0.0 + donor["head_own_share"] = np.where(borrower, head_share, 0.0) + donor["spouse_own_share"] = np.where(borrower, 1.0 - head_share, 0.0) + none = draw_scf_education_debt_households( + _features(donor), donor, seed=9, n_estimators=20 + ) + assert (none["federal_parent_plus_loan_balance"] == 0).all() -def test_imputation_reaches_only_reference_persons_and_their_spouses() -> None: +def test_imputation_reaches_only_reference_persons_spouses_and_adult_children() -> None: frame = _us_frame(_person_rows()) imputed = impute_us_scf_education_debt( frame, _ready_donor(), seed=5, n_estimators=15 @@ -563,7 +771,9 @@ def test_a_spouse_draw_without_a_spouse_moves_to_the_reference_person() -> None: donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = np.maximum( donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], 1_000.0 ) - donor["head_own_share"] = np.where(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 0, 0) + donor["head_own_share"] = 0.0 + donor["parent_plus_share"] = 0.0 + donor["child_own_share"] = 0.0 donor["spouse_own_share"] = np.where( donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 1.0, 0.0 ) @@ -583,6 +793,68 @@ def test_a_spouse_draw_without_a_spouse_moves_to_the_reference_person() -> None: assert (imputed["federal_parent_plus_loan_balance"] == 0).all() +def test_a_childs_draw_goes_to_the_youngest_adult_child() -> None: + frame = _us_frame(_person_rows(40, adult_children=True)) + donor = _ready_donor() + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = np.maximum( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], 1_000.0 + ) + for column in ("head_own_share", "spouse_own_share", "parent_plus_share"): + donor[column] = 0.0 + donor["child_own_share"] = 1.0 + donor["child_graduate_share"] = 1.0 + donor["child_is_ibr_new_borrower"] = 1.0 + donor["head_is_ibr_new_borrower"] = 0.0 + donor[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = 1.0 + imputed = impute_us_scf_education_debt(frame, donor, seed=4, n_estimators=10) + person = frame.table("person") + + single = person["person_household_id"] % 3 == 0 + youngest_child = ~single & (person["A_LINENO"] == 3) + older_child = person["A_LINENO"] == 4 + own = imputed["federal_student_loan_balance"] + plus = imputed["federal_parent_plus_loan_balance"] + # Couples have an adult child, who holds the whole draw as an own loan. + assert (own[youngest_child] > 0).all() + assert (own[~youngest_child & ~single] == 0).all() + assert (own[older_child] == 0).all() + assert ( + imputed.loc[youngest_child, "federal_student_loan_graduate_share"] == 1 + ).all() + assert imputed.loc[youngest_child, "is_income_based_repayment_new_borrower"].all() + # Single parents' children are minors: the child's draw is the head's + # Parent PLUS loan there, and nobody else in the household holds any. + single_head = single & (person["A_LINENO"] == 1) + assert (plus[single_head] > 0).all() + assert (plus[~single_head] == 0).all() + assert (own[single] == 0).all() + + +def test_adult_children_count_children_of_either_partner_aged_18_or_over() -> None: + frame = _us_frame( + pd.DataFrame.from_records( + _household( + [ + {"age": 50.0, "A_SPOUSE": 2, "A_MARITL": 1}, + {"age": 48.0, "A_SPOUSE": 1, "A_MARITL": 1, "is_female": True}, + {"age": 25.0, "PEPAR1": 2}, + {"age": 19.0, "PEPAR1": 1, "PEPAR2": 2}, + {"age": 19.0, "PEPAR1": 1}, + {"age": 12.0, "PEPAR1": 1}, + {"age": 22.0}, + ] + ) + ) + ) + recipients = education_debt._recipients(frame) + + assert recipients.features[ + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR + ].tolist() == [3.0] + # The youngest adult child, the lower line number on a tie. + assert recipients.child_position.tolist() == [3] + + def test_with_inputs_is_idempotent_and_passes_its_gate() -> None: frame = _us_frame(_person_rows()) out = with_us_scf_education_debt_inputs( @@ -693,13 +965,19 @@ def test_donor_fit_table_tabulates_observed_and_imputed_rows_alike() -> None: donor["spouse_federal_student_loan_balance"] = ( donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["spouse_own_share"] ) + donor["child_federal_student_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["child_own_share"] + ) donor["federal_parent_plus_loan_balance"] = ( donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] - donor["head_federal_student_loan_balance"] - donor["spouse_federal_student_loan_balance"] + - donor["child_federal_student_loan_balance"] ) donor["racecl4"] = (np.arange(len(donor)) % 4 + 1).astype(float) donor["spouse_age"] = donor["age"] + donor["child_borrower_age_code"] = np.where(donor["child_own_share"] > 0, 25.0, 0.0) + donor["youngest_adult_child_age_code"] = donor["child_borrower_age_code"] observed = scf_education_debt_donor_fit_table(donor) same = scf_education_debt_donor_fit_table(donor, imputed=donor) assert json.dumps(observed, sort_keys=True) == json.dumps(same, sort_keys=True) @@ -712,6 +990,17 @@ def test_donor_fit_table_tabulates_observed_and_imputed_rows_alike() -> None: assert observed["households"]["share_with_federal_education_debt"] == ( pytest.approx(expected_share) ) + # Each child borrower counts once, its weight spread over ages 18-27. + children = donor["child_own_share"] > 0 + assert children.any() + head_or_spouse = ( + (donor["head_own_share"] + donor["parent_plus_share"] > 0).astype(float) + + (donor["spouse_own_share"] > 0) + + children + ) + assert observed["borrowers"]["borrower_weight"] == pytest.approx( + float(np.dot(head_or_spouse, donor[_DONOR_WEIGHT_COLUMN])) + ) # --------------------------------------------------------------------------- # @@ -792,9 +1081,11 @@ def test_default_definition_follows_the_scf_definitions() -> None: # Income sums over the couple and their children; the mother's wages are # outside the SCF primary economic unit. assert features["employment_income"].tolist() == [90_000.0, 0.0] - assert list(features.columns) == list( - education_debt.SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS - ) + assert list(features.columns) == [ + *education_debt.SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + ] + assert features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR].tolist() == [0.0, 0.0] def test_first_receipt_definition_reproduces_the_oldest_adult_rule() -> None: diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index c7b15e3a86..6fdc510d93 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597" + "f97bbca65e9ceee16311f2879e2729c884c96af3c9d7a02b531b5b97a57e309a" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index c2d6098fcf..17644a1b04 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597" + "f97bbca65e9ceee16311f2879e2729c884c96af3c9d7a02b531b5b97a57e309a" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From b93cc3ec0a02dc0092ac86f8729057b1cdc4bd52 Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Fri, 23 Oct 2026 10:34:07 -0400 Subject: [PATCH 15/17] Keep the education-debt stage operations within the sources schema (#1073) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../build_1073_scf_education_debt_receipt.py | 35 ++++++++++++++++- .../src/microcosm/build/us/source_stages.json | 3 -- .../build/us_runtime/scf_education_debt.py | 38 +++++++++---------- .../us/test_us_bundle_core_contracts.py | 2 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/generate_us_bundle_from_constants.py | 2 +- 6 files changed, 55 insertions(+), 27 deletions(-) diff --git a/experiments/build_1073_scf_education_debt_receipt.py b/experiments/build_1073_scf_education_debt_receipt.py index bf2bd7354b..6a2401b525 100644 --- a/experiments/build_1073_scf_education_debt_receipt.py +++ b/experiments/build_1073_scf_education_debt_receipt.py @@ -523,6 +523,23 @@ def donor_under_reading( return result +def parent_plus_draw_rate(donor: pd.DataFrame, drawn: pd.DataFrame) -> float: + """Weighted share drawing Parent PLUS among held-out borrower families without it. + + Families observed with federal education debt but no Parent PLUS balance, + and drawn with debt: how often the draw gives them Parent PLUS anyway. + """ + + weight = donor["scf_weight"].to_numpy(dtype=np.float64) + observed_without = (donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0) & ( + donor["federal_parent_plus_loan_balance"] <= 0 + ) + drawn_debt = drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0 + base = (observed_without & drawn_debt).to_numpy() + plus = (drawn["federal_parent_plus_loan_balance"] > 0).to_numpy() + return float(weight[base & plus].sum() / weight[base].sum()) + + def _fold_of_family(families: np.ndarray, *, folds: int, seed: int) -> np.ndarray: unique = np.unique(families) rng = np.random.default_rng(np.random.SeedSequence([seed, 1073, 5])) @@ -1036,7 +1053,9 @@ def row(label: str, getter, formatter) -> str: "total variation distance between band distributions; MAE averages the " "absolute error of the share with debt across the groups. `educ` is the " "SCF reference person's education, which the CPS side cannot supply until " - "microcosm#297, so that row shows what the predictor would add.", + "microcosm#297, so that row shows what the predictor would add. The last " + "column is the weighted share of held-out families observed with federal " + "debt but no Parent PLUS that are drawn some Parent PLUS anyway.", "", ] metrics = [ @@ -1050,6 +1069,7 @@ def row(label: str, getter, formatter) -> str: "borrower_balance_band_tvd", "borrower_age_band_tvd", "household_rank_correlation", + "parent_plus_drawn_without_observed_parent_plus", ] lines.append("| Variant | " + " | ".join(metrics) + " |") lines.append("|---|" + "---:|" * len(metrics)) @@ -1062,7 +1082,15 @@ def row(label: str, getter, formatter) -> str: + " |" ) if host is not None: - host_metrics = [m for m in metrics if m != "household_rank_correlation"] + host_metrics = [ + m + for m in metrics + if m + not in { + "household_rank_correlation", + "parent_plus_drawn_without_observed_parent_plus", + } + ] lines += [ "", "## Host fit, before and after", @@ -1213,6 +1241,9 @@ def main(argv: Sequence[str] | None = None) -> int: observed_total, drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(dtype=np.float64), ) + metrics["parent_plus_drawn_without_observed_parent_plus"] = ( + parent_plus_draw_rate(donor, drawn) + ) receipt["holdout"]["variants"][name] = { "predictors": list(predictors), "fit": fit, diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 71e8550d47..ebefe0ee1b 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1269,8 +1269,6 @@ "x228" ], "allocation": "x7179_in_proportion_to_loans_1_6", - "whose_education_correction": "x8000_reversed_families_swap_codes_1_and_2_on_loans_4_and_6", - "child_education_holder": "co_resident_adult_child_unless_only_a_parent_could_borrow", "source": "scf_2025_full_extract" }, { @@ -1318,7 +1316,6 @@ "spouse_is_ibr_new_borrower", "child_is_ibr_new_borrower" ], - "holder_shares": "drawn_in_order_then_normalized_to_one", "weight": "wgt", "response_filter": "federal_education_debt > 0", "n_estimators": 100, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py index 83fd953ee4..dc12245ef2 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -59,25 +59,25 @@ file folds grandchild and other relative into it). The survey asks about the loans of the family living in the household, and the screener (X7801) leaves out loans of people who live elsewhere, so a code-3 loan is a - co-resident child's own loan or the parent's Parent PLUS loan. It is the child's own - (``child_federal_student_loan_balance``) when the household listing - (persons 3-12, X108-X226) has a child (code 4) aged 18 or over who could - have been in school when it was borrowed (16 or older in that year, at the - oldest age the child's public-file age band allows: the file bands listed - ages, so 18-27 all read 25), unless only a parent could have borrowed it: - an undergraduate or certificate loan whose amount borrowed exceeds the - $31,000 aggregate limit on a dependent undergraduate's own Direct Loans - (34 CFR 685.203(e)(1)), borrowed while the child could still have been - under 24, and so dependent (20 U.S.C. 1087vv(d)(1)(A)). Parent PLUS - finances only a dependent undergraduate (34 CFR 685.200(c)(2)(i)), so a - graduate loan, or one borrowed after every fitting child turned 24, is - never Parent PLUS. Every other code-3 loan is the head's - ``federal_parent_plus_loan_balance``: no listed adult child could have - borrowed it, so the student lives elsewhere and only the parent's loan is - reported, or it exceeds the dependent limit. An undergraduate loan within - the limit for a child of dependent age could be either, and is read as the - child's own, the household member the listing names; the receipt reports - the Parent PLUS share under the other reading too. + co-resident child's own loan or the parent's Parent PLUS loan. It is the + child's own (the child's ``federal_student_loan_balance``) when the + household listing (persons 3-12, X108-X226) has a child (code 4) aged 18 or + over who could have been in school when it was borrowed (16 or older in + that year, at the oldest age the child's public-file age band allows: the + file bands listed ages, so 18-27 all read 25), unless only a parent could + have borrowed it: an undergraduate or certificate loan whose amount + borrowed exceeds the $31,000 aggregate limit on a dependent undergraduate's + own Direct Loans (34 CFR 685.203(e)(1)), borrowed while the child could + still have been under 24, and so dependent (20 U.S.C. 1087vv(d)(1)(A)). + Parent PLUS finances only a dependent undergraduate (34 CFR + 685.200(c)(2)(i)), so a graduate loan, or one borrowed after every fitting + child turned 24, is never Parent PLUS. Every other code-3 loan is the + head's ``federal_parent_plus_loan_balance``: no listed adult child could + have borrowed it, so the student lives elsewhere and only the parent's loan + is reported, or it exceeds the dependent limit. An undergraduate loan + within the limit for a child of dependent age could be either, and is read + as the child's own, the household member the listing names; the receipt + reports the Parent PLUS share under the other reading too. - **IBR new-borrower status** is true when the person holds a federal loan and every federal loan the person holds (own, and for the head Parent PLUS too, since 34 CFR 685.209(b)(13)(ii) looks at any outstanding Direct or FFEL diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 5d3309e907..f6215aae11 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "f97bbca65e9ceee16311f2879e2729c884c96af3c9d7a02b531b5b97a57e309a", + "sha256": "5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 6fdc510d93..03b82ddc95 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "f97bbca65e9ceee16311f2879e2729c884c96af3c9d7a02b531b5b97a57e309a" + "5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 17644a1b04..376cfe85de 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "f97bbca65e9ceee16311f2879e2729c884c96af3c9d7a02b531b5b97a57e309a" + "5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From a5f07493919b28858b360c27f063da9ebddcabbb Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Fri, 23 Oct 2026 10:55:10 -0400 Subject: [PATCH 16/17] Re-derive the spec identity values for the child-loan routing (#1073) Regenerated on Modal (run 20261023T143414Z-b93cc3ec0): sources.yaml, the field-usage counts and source_stages claim pin, three inventory digests, the coverage pointer-inventory digest, the bundle spec_sha256 and the F0 attestation. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/evidence/spec-engine/us-f0-coverage.json | 36 +++--- .../build/spec_engine/field_usage.py | 8 +- .../build/spec_engine/inventory_coverage.py | 6 +- .../src/microcosm/build/us/spec/sources.yaml | 103 ++++++++++++++---- .../us/test_spec_engine_coverage_tool.py | 10 +- .../shared/test_spec_engine_field_usage.py | 10 +- .../us/test_us_multispine_pool_tool.py | 2 +- tools/spec_engine_coverage.py | 2 +- 8 files changed, 117 insertions(+), 60 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 2067827b8f..f1c0c3ee7a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "bd20f447db53159f719eba20c13a44efb825dc40fcc1db1d4f661ebbd6a7916b", + "documentation_sha256": "e978ca7b15daf6eacf424fcb039f63c68b7eeabc1fb0c686b2715c75686b8c89", "field_usage": { - "authored_normative_field_count": 32545, + "authored_normative_field_count": 32584, "claim_count": 49, "claims": [ { @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1845, - "pointer_sha256": "4142b1037cddd9621b70eb9677be89510474475d9a705f1cedf7a32d7bf64db8", + "pointer_count": 1884, + "pointer_sha256": "886eca845bf2e67788b19c28aa24459b9e3a5edb5f2d8129fb47d13804eca393", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42329, - "consumed_field_count": 42329, + "configuration_field_count": 42368, + "consumed_field_count": 42368, "generation0_effect_counts": { - "legacy_behavior": 38631, + "legacy_behavior": 38670, "no_generation0_effect": 3698 }, "mode_counts": { "compiler_semantic": 27729, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14143 + "legacy_behavior": 14182 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "37ce78d62b0eb4761eec78a22da637fc242c8adc516bb69ab6154430051c33e1", + "pointer_inventory_sha256": "f26245d12f4cd77a77ab4b595c9351f6923330815a87b5734e9c80e2a0a129a9", "resolved_binding_field_count": 9784, "unused_field_count": 0 }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6" + "sha256": "429406ee38dbdcf5d751a86e37004636931b69f772e346fd8c853510ba54b05d" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6" + "sha256": "429406ee38dbdcf5d751a86e37004636931b69f772e346fd8c853510ba54b05d" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", + "sha256": "ef7ed58e0170f21db8be7812f99251e702bb6b429aae463a80ea0a870c16ced5", "stages": 37 }, "failures": [], "observed": { - "sha256": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", + "sha256": "ef7ed58e0170f21db8be7812f99251e702bb6b429aae463a80ea0a870c16ced5", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043" + "sha256": "0af421c69e29c23eedd42b1d1e890b8223d637101b8b681e8fa0d07124591b72" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043" + "sha256": "0af421c69e29c23eedd42b1d1e890b8223d637101b8b681e8fa0d07124591b72" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "6fcc5e3cd9b9a4063ba211cfc499321b0e84807a60e28f76492b4e16a8a15678" + "sha256": "fec68ba41c34b4752e4bf7cbe31e35adcaa0e89be95ae522026a3a892883419e" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "90fa32603241f482d3457982f0467742e1917133d2b2a7fc120911d1ab7fcd32" + "spec_sha256": "eee022d2e258e9e7dfe6b87b80f4b731e3a679486a36abe016db5ec7515dfe10" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "90fa32603241f482d3457982f0467742e1917133d2b2a7fc120911d1ab7fcd32" + "spec_sha256": "eee022d2e258e9e7dfe6b87b80f4b731e3a679486a36abe016db5ec7515dfe10" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index db52e73774..7810324f4a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_545 +EXPECTED_AUTHORED_FIELD_COUNT = 32_584 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_784 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_329 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_368 class FieldUsageError(AssertionError): @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_845, - "4142b1037cddd9621b70eb9677be89510474475d9a705f1cedf7a32d7bf64db8", + 1_884, + "886eca845bf2e67788b19c28aa24459b9e3a5edb5f2d8129fb47d13804eca393", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index f656869dc1..03efc9093e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "72d3250c839cceaf52e6c1d24b70c229513e8eb771948865df16ab1e38142043", + "full_checkpoint": "0af421c69e29c23eedd42b1d1e890b8223d637101b8b681e8fa0d07124591b72", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "56ee2e0fcaac83d17542e0202aa4e0574ae81463768947a06d16411d035902c6", + "late_resource_semantics": "429406ee38dbdcf5d751a86e37004636931b69f772e346fd8c853510ba54b05d", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -369,7 +369,7 @@ # not run, changes its probes, so again no US solve changes. "seed_map": "0db599df419dd16779097361dc72afdefb8df2abf8b134078da9c52d31d8e017", "seed_protocol": "2b7bdc1d6b0da1bdaf54b4cac29f10f2d3c8bed2e177729c2c89e6b33a099cc5", - "source_manifest": "78a2164e5b0c62c7a534c3e53dbd26ba9761804f88f8f0f9c3e7f2d67e33377a", + "source_manifest": "ef7ed58e0170f21db8be7812f99251e702bb6b429aae463a80ea0a870c16ced5", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", } diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index ecd774c7ec..4c08305587 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 775dd27e55f382a98b8e00c17183206773fc460311997a5c1c15008466857597 + sha256: 5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b stage_manifest: version: 1 country: us @@ -1233,13 +1233,19 @@ stages: - federal_education_debt - head_federal_student_loan_balance - spouse_federal_student_loan_balance + - child_federal_student_loan_balance - federal_parent_plus_loan_balance + - parent_plus_share + - child_own_share - head_own_share - spouse_own_share - head_graduate_share - spouse_graduate_share + - child_graduate_share - head_is_ibr_new_borrower - spouse_is_ibr_new_borrower + - child_is_ibr_new_borrower + - adult_children_in_household method: scf_federal_education_loans_by_holder inputs: - x7824 @@ -1279,6 +1285,27 @@ stages: - x7905 - x7928 - x7951 + - x8000 + - x108 + - x114 + - x120 + - x126 + - x132 + - x202 + - x208 + - x214 + - x220 + - x226 + - x110 + - x116 + - x122 + - x128 + - x134 + - x204 + - x210 + - x216 + - x222 + - x228 allocation: x7179_in_proportion_to_loans_1_6 source: scf_2025_full_extract - kind: fit_weighted_qrf @@ -1306,14 +1333,19 @@ stages: - own_children_in_household - employment_income - household_income + - adult_children_in_household - federal_education_debt targets: + - parent_plus_share + - child_own_share - head_own_share - spouse_own_share - head_graduate_share - spouse_graduate_share + - child_graduate_share - head_is_ibr_new_borrower - spouse_is_ibr_new_borrower + - child_is_ibr_new_borrower weight: wgt response_filter: federal_education_debt > 0 n_estimators: 100 @@ -1324,17 +1356,24 @@ stages: - federal_parent_plus_loan_balance - is_income_based_repayment_new_borrower - federal_student_loan_graduate_share - method: allocate_household_education_debt_to_reference_person_and_spouse + method: allocate_household_education_debt_to_reference_person_spouse_and_adult_child inputs: - federal_education_debt + - parent_plus_share + - child_own_share - head_own_share - spouse_own_share - head_graduate_share - spouse_graduate_share + - child_graduate_share - head_is_ibr_new_borrower - spouse_is_ibr_new_borrower + - child_is_ibr_new_borrower - A_SPOUSE - PECOHAB + - PEPAR1 + - PEPAR2 + - age - kind: support_clip range: donor_realized outputs: @@ -1400,27 +1439,45 @@ stages: loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family''s federal share and holder split of loans 1-6. Whose education: code - 1 (the reference person) and code 2 (spouse or partner) are each person''s own federal_student_loan_balance; - code 3 (child, with grandchild and other relative folded in by the public file) is the reference person''s - federal_parent_plus_loan_balance; codes 6 (multiple people), 7 (unrelated person) and -7 (other) are - the reference person''s own balance, since none can be a Parent PLUS loan for one dependent student. - is_income_based_repayment_new_borrower is true when every federal loan the person holds was borrowed - in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014), and federal_student_loan_graduate_share - is the amount-borrowed-weighted share of the person''s own federal loans that financed a master''s, - professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all - families; a second, fit on borrower families and conditioning on the total, draws the chained holder - shares, graduate shares and new-borrower flags. Both condition on the reference person''s age, sex - and race, a spouse or partner being present (SCF MARRIED counts partners), the children of the reference - person or partner (SCF KIDS), and the wages and income less capital gains of the reference person, - spouse or partner and their children (the CPS counterpart of the SCF primary economic unit). The shared - interest-dividend and Social Security-pension predictors are left out: the CPS host carries tax-record - imputations of both, so at reference ages 25-34 42% and 18% of host households report some against - 16% and 6% of SCF families, and mapping them to the SCF distribution moves the host''s borrower share - most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). The reference person - is the CPS householder or, with a spouse or partner present, the man of a mixed-sex couple or else - the older partner; that person and the member named by its A_SPOUSE (else PECOHAB) pointer receive - the balances, and without one present the spouse''s draw goes to the reference person. Balances are - 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals.' + 1 (the reference person) and code 2 (spouse or partner) are each person''s own federal_student_loan_balance. + Where X8000 = 1 the public file reverses respondent and spouse/partner codes, but not those of loans + 4 and 6 (X7893, X7993): completed degree loans consistent with exactly one partner''s highest degree + (X5931, X6111) match the coded person on loans 1, 2, 3 and 5 and on loans 4 and 6 of unreversed families + (25 of 25), but on loans 4 and 6 of reversed families match in none of 54 (all implicates), so codes + 1 and 2 of those two loans are swapped back in reversed families. Code 3 (child, with grandchild and + other relative folded in by the public file) is a co-resident child''s own federal_student_loan_balance + when the household listing (X108-X226) has a child aged 18 or over who could have been in school in + the year borrowed (16 or older then, at the oldest age of the public file''s age band), unless only + a parent could have borrowed it: an undergraduate or certificate loan above the $31,000 aggregate + limit on a dependent undergraduate''s own Direct Loans (34 CFR 685.203(e)(1)) borrowed while the child + could still have been under 24 and so dependent (20 U.S.C. 1087vv(d)(1)(A)). Graduate loans and loans + borrowed after the child turned 24 are never Parent PLUS, which finances only a dependent undergraduate + (34 CFR 685.200(c)(2)(i)). Every other code-3 loan is the reference person''s federal_parent_plus_loan_balance: + the screener (X7801) leaves out loans of people living elsewhere, so a loan for a child not in the + household is the parent''s. Codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the + reference person''s own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower + is true when every federal loan the person holds (for the reference person, Parent PLUS too, as 34 + CFR 685.209(b)(13)(ii) looks at any Direct or FFEL balance) was borrowed in 2014 or later (the item + records a year only, so 2014 counts as after July 1, 2014); the share is an upper bound, because paid-off + pre-2014 loans are not observed and a consolidated loan is dated by its most recent underlying loan. + federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person''s own federal + loans that financed a master''s, professional or doctoral program (codes 13-15). One weighted QRF + draws the household total from all families; a second, fit on borrower families and conditioning also + on the total and the number of adult children, draws the Parent PLUS, adult child''s, reference person''s + and spouse''s shares in that order, normalized to sum to one, then the three graduate shares and new-borrower + flags. Both condition on the reference person''s age, sex and race, a spouse or partner being present + (SCF MARRIED counts partners), the children of the reference person or partner (SCF KIDS), and the + wages and income less capital gains of the reference person, spouse or partner and their children + (the CPS counterpart of the SCF primary economic unit). The shared interest-dividend and Social Security-pension + predictors are left out: the CPS host carries tax-record imputations of both, so at reference ages + 25-34 42% and 18% of host households report some against 16% and 6% of SCF families, and mapping them + to the SCF distribution moves the host''s borrower share most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). + The reference person is the CPS householder or, with a spouse or partner present, the man of a mixed-sex + couple or else the older partner; that person, the member named by its A_SPOUSE (else PECOHAB) pointer, + and the youngest child of either aged 18 or over (PEPAR1/PEPAR2) receive the balances. Without a spouse + or partner present the spouse''s draw goes to the reference person, and a child''s draw in a household + without an adult child is the reference person''s Parent PLUS loan. Balances are 2025 survey-date + dollars; the FSA calibration (microcosm#1074) pins totals.' - stage: ssi_disability_criteria survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 5e50ed272e..6a13410e86 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,21 +82,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_329 - assert fields["authored_normative_field_count"] == 32_545 + assert fields["configuration_field_count"] == 42_368 + assert fields["authored_normative_field_count"] == 32_584 assert fields["resolved_binding_field_count"] == 9_784 - assert fields["consumed_field_count"] == 42_329 + assert fields["consumed_field_count"] == 42_368 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_143, + "legacy_behavior": 14_182, "compiler_semantic": 27_729, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_631, + "legacy_behavior": 38_670, "no_generation0_effect": 3_698, } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index f87e5a9571..3efc9dbaaf 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_329 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_368 assert field_ledger.source_counts == { - "authored": 32_545, + "authored": 32_584, "resolved_bindings": 9_784, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_143, + "legacy_behavior": 14_182, "compiler_semantic": 27_729, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_631, + "legacy_behavior": 38_670, "no_generation0_effect": 3_698, } - assert len({field.pointer for field in field_ledger.fields}) == 42_329 + assert len({field.pointer for field in field_ledger.fields}) == 42_368 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index a0e723c228..1fba52f613 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -345,7 +345,7 @@ def capture_equality(expected: object, actual: object) -> None: # fit_draw_next's caller-owned uniforms (microcosm#1332) move it. # microcosm#1073's scf_wealth education-debt declaration moves it # through the bundle's source stages. - "spec_sha256": "90fa32603241f482d3457982f0467742e1917133d2b2a7fc120911d1ab7fcd32", + "spec_sha256": "eee022d2e258e9e7dfe6b87b80f4b731e3a679486a36abe016db5ec7515dfe10", }, } diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index ed3539039c..51b06f3156 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "37ce78d62b0eb4761eec78a22da637fc242c8adc516bb69ab6154430051c33e1" + "f26245d12f4cd77a77ab4b595c9351f6923330815a87b5734e9c80e2a0a129a9" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 3eb32320bab3ae0778bd32cc818088de2b56716f Mon Sep 17 00:00:00 2001 From: David Trimmer Date: Fri, 23 Oct 2026 12:58:11 -0400 Subject: [PATCH 17/17] Rerun the #1073 receipts at the merged head and benchmark Parent PLUS against FSA Modal run 20261023T152015Z-60255dd98 at 60255dd98 (tests, spec and receipt steps all passed). Parent PLUS falls from 13.8% to 9.1% of SCF federal balances and from 16.8% to 10.7% on the host; FSA's Direct Parent PLUS share is 6.7%, a floor. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../1073-scf-2025-education-debt.added.md | 2 +- .../1073-scf-education-debt-receipts.md | 134 ++- .../receipts/1073-scf-education-debt-fit.json | 1041 +++++++++++------ 3 files changed, 762 insertions(+), 415 deletions(-) diff --git a/changelog.d/1073-scf-2025-education-debt.added.md b/changelog.d/1073-scf-2025-education-debt.added.md index 21756d2721..4b32660999 100644 --- a/changelog.d/1073-scf-2025-education-debt.added.md +++ b/changelog.d/1073-scf-2025-education-debt.added.md @@ -2,7 +2,7 @@ The US build can impute federal student loan balances from the 2025 Survey of Co Only education debt reads the 2025 SCF. Net worth, the liquid-asset leaves and auto loans stay on the 2022 SCF. The `scf_wealth` stage declares the two 2025 files (`scfp2025s.zip`, `scf2025s.zip`), pinned by zip and member SHA-256 against a fresh Federal Reserve download, as artifacts 3 and 4, and a `scf_2025` vintage points at them. Moving the whole stage to 2025 later changes pinned coordinates, not code: the 2025 summary extract carries every column the shared predictor construction reads. -The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`, and codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. In families where the public file reversed respondent and spouse (X8000 = 1), the codes of loans 4 and 6 were left unreversed (none of 54 degree-discriminating loans match the coded person, against all 25 in other families), so the stage swaps them back. A loan for a child's education is the co-resident child's own loan when the household listing has a child aged 18 or over who could have been in school when it was borrowed, unless only a parent could have borrowed it: an undergraduate loan above the $31,000 limit on a dependent student's own Direct Loans, borrowed while the child could still have been dependent. Graduate loans and loans borrowed after the child turned 24 are never Parent PLUS. Every other loan for a child is the reference person's `federal_parent_plus_loan_balance`. This takes the SCF's Parent PLUS share of federal balances from 13.8% to about 9%, against 6.7% (a floor) in Federal Student Aid's portfolio. A person is an IBR new borrower when every federal loan they hold, Parent PLUS included, was borrowed in 2014 or later; that share is an upper bound, since paid-off pre-2014 loans are not observed and a consolidated loan is dated by its newest underlying loan. The graduate share weights each own loan's program by the amount borrowed. +The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`, and codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. In families where the public file reversed respondent and spouse (X8000 = 1), the codes of loans 4 and 6 were left unreversed (none of 54 degree-discriminating loans match the coded person, against all 25 in other families), so the stage swaps them back. A loan for a child's education is the co-resident child's own loan when the household listing has a child aged 18 or over who could have been in school when it was borrowed, unless only a parent could have borrowed it: an undergraduate loan above the $31,000 limit on a dependent student's own Direct Loans, borrowed while the child could still have been dependent. Graduate loans and loans borrowed after the child turned 24 are never Parent PLUS. Every other loan for a child is the reference person's `federal_parent_plus_loan_balance`. This takes the Parent PLUS share of federal balances from 13.8% to 9.1% in the SCF and from 16.8% to 10.7% on the host, against 6.7% (a floor) in Federal Student Aid's portfolio, and the share of borrowers aged 24 or younger from 5.0% to 12.2% on the host (FSA: 13.8%). A person is an IBR new borrower when every federal loan they hold, Parent PLUS included, was borrowed in 2014 or later; that share is an upper bound, since paid-off pre-2014 loans are not observed and a consolidated loan is dated by its newest underlying loan. The graduate share weights each own loan's program by the amount borrowed. The model. One weighted QRF draws each household's federal education debt. A second, fit on borrower families and conditioning on that total and the number of adult children, draws the Parent PLUS, adult child's, reference person's and spouse's shares in that order, normalized to sum to one, then the graduate shares and new-borrower flags. Parent PLUS is drawn for itself, so a family like the donors without it rarely draws it. Both condition on the reference person's age, sex and race, a spouse or partner being present, the children of the reference person or partner, and the wages and income less capital gains of the reference person, spouse or partner and their children. The CPS side codes each predictor the way the SCF defines it: SCF `MARRIED` counts cohabiting partners, SCF `KIDS` counts the partner's children, the reference person comes from the householder couple (the man, or the older partner), and income covers the primary economic unit rather than the whole household. The shared interest-dividend and Social Security-pension predictors are left out (`SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS`). The CPS host carries tax-record imputations of both, so at reference ages 25-34 it reports them for 42% and 18% of households against 16% and 6% of SCF families, and that shift was the main reason the first host fit drew too few borrowers. The reference person, the member its `A_SPOUSE` (else `PECOHAB`) pointer names, and the youngest child of either aged 18 or over receive the balances; a child's draw in a household without an adult child is the reference person's Parent PLUS loan. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. diff --git a/experiments/1073-scf-education-debt-receipts.md b/experiments/1073-scf-education-debt-receipts.md index cbc6d6a5e4..767713741f 100644 --- a/experiments/1073-scf-education-debt-receipts.md +++ b/experiments/1073-scf-education-debt-receipts.md @@ -1,10 +1,10 @@ # microcosm#1073 receipts: SCF 2025 federal student loan balances -Generated 2026-10-22 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`, commit `2a6e65a739b79a8232e1177610d11928fe92e26c`, Modal run `20261022T153020Z-2a6e65a73` (`experiments/modal_1073.py`). Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. +Generated 2026-10-23 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`, commit `60255dd9808ff1a0918e8c781b8818ce3376c32c`, Modal run `20261023T152015Z-60255dd98` (`experiments/modal_1073.py`). Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. Host columns impute the published default `populace-us-2024-spm-20260915` at its calibrated weights under three settings: the first receipt's construction (oldest adult, `A_MARITL` marriage, own children, whole-household income, every candidate predictor); the SCF-matching codings with every candidate predictor; and the build, which also leaves out interest and dividends and Social Security and pensions. The cause and the evidence are in the stage notes and `experiments/receipts/1073-predictor-shift-diagnosis.json`. -Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5). Household figures group by the reference person's age and race and by weighted quintiles of income less capital gains of the setting's income unit, the income concept that setting conditions on. Borrower figures count people with a federal balance (own plus Parent PLUS), in Federal Student Aid's balance and age bands; SCF borrowers are the reference person and spouse. Nothing here is a gate or a calibration target. +Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5). Household figures group by the reference person's age and race and by weighted quintiles of income less capital gains of the setting's income unit, the income concept that setting conditions on. Borrower figures count people with a federal balance (own plus Parent PLUS), in Federal Student Aid's balance and age bands; SCF borrowers are the reference person, the spouse or partner and an adult child holding a loan for their own education, whose public-file age band (18-27 reads 25) is spread evenly over its years. Nothing here is a gate or a calibration target. ## Source checks @@ -21,7 +21,7 @@ Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5 | non federal of loans 1 6 | 12.0% | | federal status blank of loans 1 6 | 0.6% | -| Federal balances on loans 1-6, by whose education | Share | +| Federal balances on loans 1-6, by whose education (codes as published) | Share | |---|---:| | 1 respondent (reference person) | 54.8% | | 2 spouse or partner | 31.4% | @@ -30,44 +30,92 @@ Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5 | 7 unrelated person | 0.0% | | -7 other | 0.0% | +### X8000: are the whose-education codes reversed? + +Completed degree loans coded 1 or 2 in families with a spouse or partner. A loan discriminates when it is consistent with exactly one partner's highest degree (X5931, X6111: at least the program's level); the last columns count those consistent with the coded person (code 1 the reference person, code 2 the spouse or partner), as published, before the stage swaps codes 1 and 2 of loans 4 and 6 in reversed families. Counts pool the five implicates (implicate 1 in brackets). + +| Loans | X8000 | Code | Discriminating | Consistent with the coded person | +|---|---|---:|---:|---:| +| 1, 2, 3, 5 | not reversed | 1 | 180 (36) | 175 (35) | +| 1, 2, 3, 5 | not reversed | 2 | 208 (40) | 203 (40) | +| 1, 2, 3, 5 | reversed | 1 | 73 (15) | 70 (14) | +| 1, 2, 3, 5 | reversed | 2 | 470 (92) | 470 (92) | +| 4, 6 | not reversed | 1 | 20 (4) | 20 (4) | +| 4, 6 | not reversed | 2 | 5 (1) | 5 (1) | +| 4, 6 | reversed | 1 | 49 (9) | 0 (0) | +| 4, 6 | reversed | 2 | 5 (1) | 0 (0) | + +### Loans for a child's education (SCF 2025, weighted) + +FSA's Direct Parent PLUS balance is 6.7% of its portfolio at FY2026 Q3, a floor, since consolidated Parent PLUS loans sit in the consolidation total. + +| Reading | Parent PLUS share | Child's own share | Parent PLUS holders | Child borrowers | Borrowers | +|---|---:|---:|---:|---:|---:| +| Before: every code-3 loan Parent PLUS, codes as published | 13.8% | 0.0% | 4.4M | 0.0M | 27.2M | +| X8000 correction only | 13.8% | 0.0% | 4.4M | 0.0M | 26.9M | +| Build: co-resident adult child unless only a parent could borrow | 9.1% | 4.7% | 2.1M | 2.3M | 27.2M | +| Alternative: within-limit undergraduate loans for a dependent-age child also Parent PLUS | 11.8% | 2.0% | 4.1M | 0.4M | 27.1M | + +IBR new-borrower status is an upper bound: the SCF sees only loans still owed, so paid-off pre-2014 loans are missed, and a consolidated loan is dated by its most recent underlying loan. 29.3% of federal balances dated 2014 or later are on a loan that has been consolidated or refinanced (X9320-X9325). + +## Parent PLUS and borrowers against Federal Student Aid + +Before is the previous rule (every loan for a child's education the head's Parent PLUS loan, the residual of the composition draw); the host's before column is the build in the previous receipt (commit `2a6e65a73`, Modal run `20261022T153020Z-2a6e65a73`). FSA's Parent PLUS share is a floor; its dollars are the whole federal portfolio, of which household surveys capture part, so the shares compare and the levels do not. FSA borrowers come from its by-age table (FY2026 Q2), which counts the parent on a Parent PLUS loan; its 42.3M unduplicated recipients (FY2026 Q3) count the student instead. + +| | FSA | SCF 2025, before | SCF 2025 | Host: Build, before | Host: Build | +|---|---:|---:|---:|---:|---:| +| Parent PLUS share of federal balances | 6.7% (floor) | 13.8% | 9.1% | 16.8% | 10.7% | +| Parent PLUS balances | $115.0B | $138.3B | $91.2B | $143.9B | $91.8B | +| Federal balances | $1,724.5B | $1,005.9B | $1,005.9B | $858.4B | $858.4B | +| Borrowers (people) | 44.9M | 27.2M | 27.2M | 21.2M | 21.2M | +| Borrowers aged 24 and younger | 13.8% | 5.7% | 10.9% | 5.0% | 12.2% | +| Borrowers aged 25 to 34 | 31.2% | 33.1% | 35.3% | 29.2% | 31.1% | +| Borrowers aged 35 to 49 | 33.5% | 38.1% | 37.2% | 37.8% | 37.2% | +| Borrowers aged 50 to 61 | 14.3% | 16.2% | 11.2% | 20.8% | 13.7% | +| Borrowers aged 62 and older | 7.1% | 6.9% | 5.4% | 7.2% | 5.8% | + ## Fit | | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | |---|---:|---:|---:|---:|---:| | Households with federal education debt | 18.2% | 17.9% | 14.4% | 14.5% | 16.4% | -| Mean among them | $41,052 | $38,130 | $44,483 | $42,174 | $42,680 | +| Mean among them | $41,052 | $38,156 | $44,483 | $42,174 | $42,680 | | Median among them | $25,000 | $23,000 | $25,000 | $25,000 | $23,000 | | Weighted federal education debt | $1.006T | $0.919T | $0.786T | $0.747T | $0.858T | -| Parent PLUS share of the total | 13.8% | 12.2% | 21.5% | 15.5% | 16.8% | -| Borrowers (people) | 27.2M | 25.7M | 18.5M | 18.5M | 21.2M | -| Mean balance per borrower | $36,934 | $35,733 | $42,554 | $40,307 | $40,447 | -| Median balance per borrower | $21,000 | $21,000 | $24,000 | $24,000 | $22,000 | -| IBR new borrowers among borrowers | 59.2% | 59.3% | 60.6% | 61.7% | 61.5% | -| Graduate share of own original principal | 47.7% | 45.0% | 53.6% | 49.8% | 48.3% | +| Parent PLUS share of the total | 9.1% | 6.8% | 13.0% | 10.7% | 10.7% | +| Borrowers (people) | 27.2M | 25.8M | 18.8M | 18.6M | 21.2M | +| Mean balance per borrower | $36,931 | $35,618 | $41,827 | $40,063 | $40,535 | +| Median balance per borrower | $21,000 | $21,000 | $24,700 | $24,000 | $22,224 | +| IBR new borrowers among borrowers (upper bound) | 59.5% | 58.4% | 57.9% | 59.7% | 61.3% | +| Graduate share of own original principal | 47.2% | 44.7% | 50.8% | 48.8% | 50.0% | ### Borrowers by FSA balance band -| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | -|---|---:|---:|---:|---:|---:| -| under_5k | 11.9% | 10.2% | 10.4% | 11.8% | 12.5% | -| 5k_to_10k | 15.5% | 16.6% | 16.7% | 15.4% | 13.6% | -| 10k_to_20k | 18.6% | 18.7% | 15.5% | 16.1% | 19.0% | -| 20k_to_40k | 22.3% | 23.6% | 23.0% | 23.8% | 23.5% | -| 40k_to_60k | 12.8% | 12.0% | 10.4% | 11.5% | 10.8% | -| 60k_to_80k | 8.1% | 8.6% | 9.6% | 9.1% | 8.7% | -| 80k_to_100k | 3.7% | 3.9% | 5.1% | 3.9% | 3.2% | -| 100k_to_200k | 4.9% | 4.7% | 6.4% | 5.5% | 5.6% | -| 200k_plus | 2.2% | 1.6% | 2.9% | 2.9% | 3.1% | +FSA: borrowers by debt size, FY2026 Q2. + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | FSA | +|---|---:|---:|---:|---:|---:|---:| +| under_5k | 11.9% | 10.5% | 11.0% | 11.8% | 12.8% | 15.4% | +| 5k_to_10k | 15.5% | 16.1% | 15.8% | 15.2% | 13.7% | 16.1% | +| 10k_to_20k | 18.2% | 18.4% | 15.1% | 15.9% | 18.1% | 20.5% | +| 20k_to_40k | 22.8% | 24.0% | 23.9% | 24.4% | 23.9% | 21.2% | +| 40k_to_60k | 13.0% | 12.6% | 10.3% | 11.4% | 10.7% | 9.4% | +| 60k_to_80k | 7.8% | 8.6% | 9.7% | 9.1% | 9.0% | 5.8% | +| 80k_to_100k | 3.7% | 3.7% | 5.0% | 3.6% | 3.4% | 3.1% | +| 100k_to_200k | 4.9% | 4.4% | 6.3% | 5.7% | 5.2% | 5.8% | +| 200k_plus | 2.2% | 1.7% | 2.9% | 3.0% | 3.3% | 2.7% | ### Borrowers by FSA age band -| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | -|---|---:|---:|---:|---:|---:| -| 24_and_younger | 5.7% | 6.5% | 3.8% | 5.5% | 5.0% | -| 25_to_34 | 33.1% | 33.9% | 26.6% | 30.9% | 29.2% | -| 35_to_49 | 38.1% | 38.0% | 39.8% | 37.3% | 37.8% | -| 50_to_61 | 16.2% | 15.1% | 21.1% | 19.4% | 20.8% | -| 62_and_older | 6.9% | 6.6% | 8.7% | 6.9% | 7.2% | +FSA: borrowers by age, FY2026 Q2, of those with an age reported. + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | FSA | +|---|---:|---:|---:|---:|---:|---:| +| 24_and_younger | 10.9% | 10.9% | 11.7% | 11.9% | 12.2% | 13.8% | +| 25_to_34 | 35.3% | 35.9% | 30.3% | 31.8% | 31.1% | 31.2% | +| 35_to_49 | 37.2% | 36.3% | 37.2% | 37.3% | 37.2% | 33.5% | +| 50_to_61 | 11.2% | 11.0% | 13.9% | 13.3% | 13.7% | 14.3% | +| 62_and_older | 5.4% | 5.9% | 6.8% | 5.8% | 5.8% | 7.1% | ### Households with federal education debt, by age of reference person @@ -85,9 +133,9 @@ Median among borrowers: |---|---:|---:|---:|---:|---:| | 24_and_younger | $10,000 | $14,000 | $6,000 | $14,000 | $12,662 | | 25_to_34 | $24,900 | $23,000 | $20,200 | $21,600 | $20,100 | -| 35_to_49 | $25,000 | $22,900 | $25,670 | $25,000 | $23,000 | +| 35_to_49 | $25,000 | $22,000 | $25,670 | $25,000 | $23,000 | | 50_to_61 | $28,000 | $27,000 | $29,000 | $30,000 | $28,742 | -| 62_and_older | $26,000 | $26,997 | $38,000 | $26,000 | $34,000 | +| 62_and_older | $26,000 | $27,000 | $38,000 | $26,000 | $34,000 | ### Households with federal education debt, by race (`racecl4`) @@ -129,16 +177,16 @@ Median among borrowers: ## Predictor ablation (five-fold hold-out on SCF families) -Each variant refits both QRFs. Errors are drawn minus observed; TVD is total variation distance between band distributions; MAE averages the absolute error of the share with debt across the groups. `educ` is the SCF reference person's education, which the CPS side cannot supply until microcosm#297, so that row shows what the predictor would add. +Each variant refits both QRFs. Errors are drawn minus observed; TVD is total variation distance between band distributions; MAE averages the absolute error of the share with debt across the groups. `educ` is the SCF reference person's education, which the CPS side cannot supply until microcosm#297, so that row shows what the predictor would add. The last column is the weighted share of held-out families observed with federal debt but no Parent PLUS that are drawn some Parent PLUS anyway. -| Variant | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | household_rank_correlation | -|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| shared_8_without_age | -0.250 | -1.737 | -4.000 | 0.022 | 4.812 | 0.648 | 0.672 | 0.028 | 0.107 | 0.154 | -| shared_8 | -0.532 | -4.092 | -8.000 | 0.041 | 1.193 | 1.078 | 0.876 | 0.037 | 0.016 | 0.173 | -| shared_8_plus_household_income | -0.873 | -4.350 | -8.000 | 0.037 | 1.351 | 1.359 | 0.923 | 0.030 | 0.009 | 0.175 | -| plus_household_income_and_education | -1.474 | -5.567 | -4.816 | 0.040 | 1.321 | 2.067 | 1.474 | 0.050 | 0.023 | 0.208 | -| build_without_interest_and_ss_pension (build) | -0.304 | -7.119 | -8.000 | 0.032 | 1.260 | 1.069 | 0.675 | 0.033 | 0.015 | 0.168 | -| build_plus_education | -0.920 | -7.087 | -8.000 | 0.028 | 1.749 | 1.789 | 0.920 | 0.029 | 0.031 | 0.207 | +| Variant | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | household_rank_correlation | parent_plus_drawn_without_observed_parent_plus | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| shared_8_without_age | -0.250 | -1.707 | -4.000 | 0.023 | 4.812 | 0.648 | 0.672 | 0.033 | 0.122 | 0.154 | 0.046 | +| shared_8 | -0.532 | -4.099 | -8.000 | 0.041 | 1.193 | 1.078 | 0.876 | 0.037 | 0.020 | 0.173 | 0.025 | +| shared_8_plus_household_income | -0.873 | -4.378 | -8.000 | 0.036 | 1.351 | 1.359 | 0.923 | 0.032 | 0.009 | 0.175 | 0.027 | +| plus_household_income_and_education | -1.474 | -5.582 | -4.816 | 0.040 | 1.321 | 2.067 | 1.474 | 0.052 | 0.031 | 0.208 | 0.016 | +| build_without_interest_and_ss_pension (build) | -0.304 | -7.054 | -8.000 | 0.030 | 1.260 | 1.069 | 0.675 | 0.027 | 0.011 | 0.168 | 0.030 | +| build_plus_education | -0.920 | -7.077 | -8.000 | 0.028 | 1.749 | 1.789 | 0.920 | 0.035 | 0.038 | 0.207 | 0.024 | ## Host fit, before and after @@ -146,7 +194,7 @@ Imputed with build seed 0 onto `populace-us-2024-spm-20260915` (57,240 household | Setting | Households with debt | Parent PLUS share | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| First receipt | 14.4% | 21.5% | -3.742 | 8.357 | 0.000 | 0.062 | 3.554 | 2.834 | 4.252 | 0.070 | 0.085 | -| SCF codings, all predictors | 14.5% | 15.5% | -3.701 | 2.732 | 0.000 | 0.040 | 4.170 | 2.570 | 4.568 | 0.040 | 0.032 | -| Build | 16.4% | 16.8% | -1.741 | 3.966 | -8.000 | 0.046 | 2.391 | 0.942 | 2.866 | 0.045 | 0.050 | -| SCF 2025 | 18.2% | 13.8% | | | | | | | | | | +| First receipt | 14.4% | 13.0% | -3.742 | 8.357 | 0.000 | 0.062 | 3.554 | 2.834 | 4.252 | 0.066 | 0.050 | +| SCF codings, all predictors | 14.5% | 10.7% | -3.701 | 2.732 | 0.000 | 0.040 | 4.170 | 2.570 | 4.568 | 0.044 | 0.035 | +| Build | 16.4% | 10.7% | -1.741 | 3.966 | -8.000 | 0.046 | 2.391 | 0.942 | 2.866 | 0.046 | 0.042 | +| SCF 2025 | 18.2% | 9.1% | | | | | | | | | | diff --git a/experiments/receipts/1073-scf-education-debt-fit.json b/experiments/receipts/1073-scf-education-debt-fit.json index bced2dbead..4028104efa 100644 --- a/experiments/receipts/1073-scf-education-debt-fit.json +++ b/experiments/receipts/1073-scf-education-debt-fit.json @@ -1,10 +1,10 @@ { "schema_version": 1, "issue": "PolicyEngine/microcosm#1073", - "generated_at": "2026-10-22", + "generated_at": "2026-10-23", "provenance": { - "commit": "2a6e65a739b79a8232e1177610d11928fe92e26c", - "run": "20261022T153020Z-2a6e65a73" + "commit": "60255dd9808ff1a0918e8c781b8818ce3376c32c", + "run": "20261023T152015Z-60255dd98" }, "scf_files": { "rscfp2025.dta": { @@ -44,9 +44,302 @@ }, "whose_education_codes_outside_the_rule": [], "federal_education_debt_after_x7179_rule": 1005887392198.3959, - "remaining_loans_without_detail_dollars": 0.0 + "remaining_loans_without_detail_dollars": 0.0, + "x8000_degree_check": [ + { + "loans": "1, 2, 3, 5", + "x8000": "not reversed", + "code": 1, + "discriminating": 180, + "consistent_with_coded_person": 175, + "discriminating_implicate_1": 36, + "consistent_implicate_1": 35 + }, + { + "loans": "1, 2, 3, 5", + "x8000": "not reversed", + "code": 2, + "discriminating": 208, + "consistent_with_coded_person": 203, + "discriminating_implicate_1": 40, + "consistent_implicate_1": 40 + }, + { + "loans": "1, 2, 3, 5", + "x8000": "reversed", + "code": 1, + "discriminating": 73, + "consistent_with_coded_person": 70, + "discriminating_implicate_1": 15, + "consistent_implicate_1": 14 + }, + { + "loans": "1, 2, 3, 5", + "x8000": "reversed", + "code": 2, + "discriminating": 470, + "consistent_with_coded_person": 470, + "discriminating_implicate_1": 92, + "consistent_implicate_1": 92 + }, + { + "loans": "4, 6", + "x8000": "not reversed", + "code": 1, + "discriminating": 20, + "consistent_with_coded_person": 20, + "discriminating_implicate_1": 4, + "consistent_implicate_1": 4 + }, + { + "loans": "4, 6", + "x8000": "not reversed", + "code": 2, + "discriminating": 5, + "consistent_with_coded_person": 5, + "discriminating_implicate_1": 1, + "consistent_implicate_1": 1 + }, + { + "loans": "4, 6", + "x8000": "reversed", + "code": 1, + "discriminating": 49, + "consistent_with_coded_person": 0, + "discriminating_implicate_1": 9, + "consistent_implicate_1": 0 + }, + { + "loans": "4, 6", + "x8000": "reversed", + "code": 2, + "discriminating": 5, + "consistent_with_coded_person": 0, + "discriminating_implicate_1": 1, + "consistent_implicate_1": 0 + } + ], + "child_education_readings": { + "before": { + "parent_plus_share_of_federal": 0.13750075307389345, + "child_own_share_of_federal": 0.0, + "parent_plus_dollars": 138310273934.81424, + "borrowers": 27234421.461229563, + "parent_plus_holders": 4381038.706092626, + "child_borrowers": 0.0 + }, + "x8000_only": { + "parent_plus_share_of_federal": 0.13750075307389345, + "child_own_share_of_federal": 0.0, + "parent_plus_dollars": 138310273934.81424, + "borrowers": 26923887.076506052, + "parent_plus_holders": 4381038.706092626, + "child_borrowers": 0.0 + }, + "build": { + "parent_plus_share_of_federal": 0.09068330207742185, + "child_own_share_of_federal": 0.04681745099647159, + "parent_plus_dollars": 91217190242.59724, + "borrowers": 27237098.83004764, + "parent_plus_holders": 2116082.8804507507, + "child_borrowers": 2347267.516216494 + }, + "within_limit_parent_plus": { + "parent_plus_share_of_federal": 0.11789995879009203, + "child_own_share_of_federal": 0.019600794283801418, + "parent_plus_dollars": 118594082087.66402, + "borrowers": 27076002.732565746, + "parent_plus_holders": 4144863.2081744918, + "child_borrowers": 388291.15397783055 + } + }, + "consolidated_share_of_federal_balances_dated_2014_or_later": 0.2928368593828854 + }, + "fsa": { + "files": { + "PortfolioSummary.xls": "11606d4b93ceea969fcae0f790f3a4e1c2999b62050559ce89bdc269810a62ee", + "PortfoliobyLoanType.xls": "e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33", + "Portfolio-by-Age.xls": "c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef", + "Portfolio-by-Debt-Size.xls": "881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d" + }, + "total_dollars_fy2026_q3": 1724500000000.0, + "unduplicated_recipients_fy2026_q3": 42300000.0, + "parent_plus_dollars_fy2026_q3": 115000000000.0, + "parent_plus_recipients_fy2026_q3": 3500000.0, + "borrowers_by_age_fy2026_q2": { + "24_and_younger": 6200000.0, + "25_to_34": 14000000.0, + "35_to_49": 15000000.0, + "50_to_61": 6400000.0, + "62_and_older": 3200000.0, + "not_reported": 100000.0 + }, + "borrowers_by_debt_size_fy2026_q2": { + "under_5k": 6900000.0, + "5k_to_10k": 7200000.0, + "10k_to_20k": 9200000.0, + "20k_to_40k": 9500000.0, + "40k_to_60k": 4200000.0, + "60k_to_80k": 2600000.0, + "80k_to_100k": 1400000.0, + "100k_to_200k": 2600000.0, + "200k_plus": 1200000.0 + } + }, + "previous_receipt": { + "commit": "2a6e65a739b79a8232e1177610d11928fe92e26c", + "run": "20261022T153020Z-2a6e65a73", + "host_build": { + "parent_plus_share_of_total": 0.16758194869071996, + "federal_education_debt_total": 858421208954.9309, + "borrower_weight": 21223403.433088087, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257 + } + } }, "scf_2025_observed": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.18154956812200357, + "mean_among_borrowers": 41052.27209260245, + "median_among_borrowers": 25000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 1005887392198.3959, + "parent_plus_share_of_total": 0.09068330207742185 + }, + "household_balance_bands": { + "under_5k": 0.09639294713632403, + "5k_to_10k": 0.14003280730185305, + "10k_to_20k": 0.1763887243762958, + "20k_to_40k": 0.23167145676213327, + "40k_to_60k": 0.13228448002879964, + "60k_to_80k": 0.09356242722856564, + "80k_to_100k": 0.04781176224843039, + "100k_to_200k": 0.05847272768581226, + "200k_plus": 0.023382667231785843 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.22183897089190358, + "mean_among_borrowers": 23531.319738797185, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3615365625657711, + "mean_among_borrowers": 36363.72833498089, + "median_among_borrowers": 24900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.27186430754586627, + "mean_among_borrowers": 45297.2739464529, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.1666515695303606, + "mean_among_borrowers": 44901.11056030461, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.041870913769168856, + "mean_among_borrowers": 41544.318165395336, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.17240693757923897, + "mean_among_borrowers": 41150.07823437558, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.24343266231667623, + "mean_among_borrowers": 47375.69026752636, + "median_among_borrowers": 30000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.1409034533516125, + "mean_among_borrowers": 33382.63507234893, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.19450371139945485, + "mean_among_borrowers": 37451.352530441625, + "median_among_borrowers": 20000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.1144757325207629, + "mean_among_borrowers": 30747.429065421915, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.1705804998024808, + "mean_among_borrowers": 41222.638862750704, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17774016899539674, + "mean_among_borrowers": 38756.18543238133, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26595688145561025, + "mean_among_borrowers": 40379.14671664771, + "median_among_borrowers": 28000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.1790121945626286, + "mean_among_borrowers": 50757.93410007803, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 27237098.830047637, + "mean_balance": 36930.783211342365, + "median_balance": 21000.0, + "balance_bands": { + "under_5k": 0.11858724248854419, + "5k_to_10k": 0.15545017628617694, + "10k_to_20k": 0.18221344662589709, + "20k_to_40k": 0.22815323048913974, + "40k_to_60k": 0.12964994896026388, + "60k_to_80k": 0.07784527443403214, + "80k_to_100k": 0.03718546813313425, + "100k_to_200k": 0.04901089121852003, + "200k_plus": 0.021904321364291858 + }, + "age_bands": { + "24_and_younger": 0.10943214673515073, + "25_to_34": 0.3533895334655874, + "35_to_49": 0.37160638234348325, + "50_to_61": 0.11180436490051585, + "62_and_older": 0.05376757255526285 + }, + "graduate_share_of_own_original_principal": 0.47206971539270537, + "ibr_new_borrower_share": 0.5952213780001776 + } + }, + "scf_2025_observed_before": { "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.18154956812200357, @@ -203,18 +496,18 @@ "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.17905210701848603, - "mean_among_borrowers": 40339.111086866, + "mean_among_borrowers": 40351.49510409681, "median_among_borrowers": 24000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 974816127786.9578, - "parent_plus_share_of_total": 0.12332712997930678 + "federal_education_debt_total": 975115394166.3627, + "parent_plus_share_of_total": 0.06206739011401023 }, "household_balance_bands": { "under_5k": 0.10118107773912553, - "5k_to_10k": 0.13505410339066057, - "10k_to_20k": 0.18318555601050834, - "20k_to_40k": 0.21964423310630496, - "40k_to_60k": 0.13583862553597464, + "5k_to_10k": 0.13503432715253297, + "10k_to_20k": 0.18320533224863594, + "20k_to_40k": 0.21890176642533937, + "40k_to_60k": 0.13658109221694026, "60k_to_80k": 0.1008575608439786, "80k_to_100k": 0.045400028448036055, "100k_to_200k": 0.05696009741342364, @@ -230,25 +523,25 @@ "25_to_34": { "household_weight": 20594001.0, "share_with_federal_education_debt": 0.2634230533073922, - "mean_among_borrowers": 39081.51087614914, + "mean_among_borrowers": 39094.6586979493, "median_among_borrowers": 25000.0 }, "35_to_49": { "household_weight": 34447794.528511114, "share_with_federal_education_debt": 0.2511547364155009, - "mean_among_borrowers": 41977.825875862174, + "mean_among_borrowers": 42005.58484858575, "median_among_borrowers": 25000.0 }, "50_to_61": { "household_weight": 26383416.705058806, "share_with_federal_education_debt": 0.20137196508720936, - "mean_among_borrowers": 39932.6451255711, + "mean_among_borrowers": 39931.548888414545, "median_among_borrowers": 23000.0 }, "62_and_older": { "household_weight": 47684957.45658214, "share_with_federal_education_debt": 0.06509257053181323, - "mean_among_borrowers": 42570.021847555945, + "mean_among_borrowers": 42567.96041221, "median_among_borrowers": 28000.0 } }, @@ -256,25 +549,25 @@ "white_non_hispanic": { "household_weight": 81838505.71507175, "share_with_federal_education_debt": 0.16722261212813858, - "mean_among_borrowers": 39982.38559368015, - "median_among_borrowers": 24000.0 + "mean_among_borrowers": 40003.15075854964, + "median_among_borrowers": 24306.0 }, "black_non_hispanic": { "household_weight": 18706038.359267425, "share_with_federal_education_debt": 0.236098978018357, - "mean_among_borrowers": 45093.93722051315, + "mean_among_borrowers": 45086.614893713966, "median_among_borrowers": 27000.0 }, "hispanic": { "household_weight": 15955872.224651583, "share_with_federal_education_debt": 0.15009122535417432, - "mean_among_borrowers": 34558.186278806694, + "mean_among_borrowers": 34555.5308698397, "median_among_borrowers": 20000.0 }, "other_or_multiple": { "household_weight": 18463274.3911613, "share_with_federal_education_debt": 0.1987172729932262, - "mean_among_borrowers": 39719.50100870802, + "mean_among_borrowers": 39734.161271935416, "median_among_borrowers": 20200.0 } }, @@ -282,71 +575,72 @@ "1": { "household_weight": 26990442.923403554, "share_with_federal_education_debt": 0.1250378708596092, - "mean_among_borrowers": 32495.496996203394, + "mean_among_borrowers": 32497.713789742633, "median_among_borrowers": 19200.0 }, "2": { "household_weight": 26994482.864333212, "share_with_federal_education_debt": 0.16848970888296705, - "mean_among_borrowers": 40638.90333292065, + "mean_among_borrowers": 40638.8255469805, "median_among_borrowers": 23000.0 }, "3": { "household_weight": 26997020.193277217, "share_with_federal_education_debt": 0.17317139260393358, - "mean_among_borrowers": 38283.29263303069, - "median_among_borrowers": 23900.0 + "mean_among_borrowers": 38343.90461862453, + "median_among_borrowers": 24000.0 }, "4": { "household_weight": 26985790.839470405, "share_with_federal_education_debt": 0.26089759644033866, - "mean_among_borrowers": 43703.342247747096, + "mean_among_borrowers": 43703.27046192123, "median_among_borrowers": 26000.0 }, "5": { "household_weight": 26995953.869667668, "share_with_federal_education_debt": 0.16768340855350833, - "mean_among_borrowers": 42776.28156983026, - "median_among_borrowers": 27000.0 + "mean_among_borrowers": 42778.330819886694, + "median_among_borrowers": 26000.0 } }, "borrowers": { - "borrower_weight": 25995647.800317153, - "mean_balance": 37499.20506982191, - "median_balance": 22000.0, + "borrower_weight": 25895695.700705215, + "mean_balance": 37655.50095415307, + "median_balance": 23000.0, "balance_bands": { - "under_5k": 0.12187825205948413, - "5k_to_10k": 0.1410220365125263, - "10k_to_20k": 0.1851654547328621, - "20k_to_40k": 0.21348131319748645, - "40k_to_60k": 0.13547790411800253, - "60k_to_80k": 0.09385336282040968, - "80k_to_100k": 0.04113173480343179, - "100k_to_200k": 0.0501832187776321, - "200k_plus": 0.017806722978165053 + "under_5k": 0.11923381414468255, + "5k_to_10k": 0.13703997807327198, + "10k_to_20k": 0.18638889289819083, + "20k_to_40k": 0.21794950848449265, + "40k_to_60k": 0.1381837484113108, + "60k_to_80k": 0.09092222085828328, + "80k_to_100k": 0.03949213571886882, + "100k_to_200k": 0.0528776909925517, + "200k_plus": 0.017912010418347366 }, "age_bands": { - "24_and_younger": 0.07113074708615924, - "25_to_34": 0.24900499283093128, - "35_to_49": 0.3562472636589869, - "50_to_61": 0.21093410606503074, - "62_and_older": 0.11268289035889188 - }, - "graduate_share_of_own_original_principal": 0.4521386518636319, - "ibr_new_borrower_share": 0.5904131759381358 + "24_and_younger": 0.12221762793944202, + "25_to_34": 0.2714652723519627, + "35_to_49": 0.3314329221489056, + "50_to_61": 0.17835091169774608, + "62_and_older": 0.09653326586194358 + }, + "graduate_share_of_own_original_principal": 0.4503453183190178, + "ibr_new_borrower_share": 0.5833971357123033 } }, "metrics": { "share_with_debt_error_pp": -0.24974611035175343, - "mean_among_borrowers_error_pct": -1.7372022774470364, + "mean_among_borrowers_error_pct": -1.7070358174692046, "median_among_borrowers_error_pct": -4.0000000000000036, - "household_balance_band_tvd": 0.022434241359602005, + "household_balance_band_tvd": 0.023196484278695206, "head_age_band_share_mae_pp": 4.81151166418894, "race_share_mae_pp": 0.6479835836438199, "income_quintile_share_mae_pp": 0.6721955334843016, - "borrower_balance_band_tvd": 0.028340960926920844, - "borrower_age_band_tvd": 0.10686848931450572, - "household_rank_correlation": 0.15376545505926326 + "borrower_balance_band_tvd": 0.03260623116349648, + "borrower_age_band_tvd": 0.1220977213082023, + "household_rank_correlation": 0.15376716764930412, + "parent_plus_drawn_without_observed_parent_plus": 0.04628009643969982 } }, "shared_8": { @@ -364,18 +658,18 @@ "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.17622772670671366, - "mean_among_borrowers": 39372.39907723004, + "mean_among_borrowers": 39369.48956513442, "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 936446699439.1069, - "parent_plus_share_of_total": 0.12480448512425425 + "federal_education_debt_total": 936377498601.3939, + "parent_plus_share_of_total": 0.07131444648553982 }, "household_balance_bands": { - "under_5k": 0.09343877143258432, - "5k_to_10k": 0.16316715085533556, + "under_5k": 0.09351058185138737, + "5k_to_10k": 0.1630953404365325, "10k_to_20k": 0.1722357902778136, - "20k_to_40k": 0.22990980510936418, - "40k_to_60k": 0.12014043404307863, + "20k_to_40k": 0.23003959537160001, + "40k_to_60k": 0.12001064378084277, "60k_to_80k": 0.1082920780850408, "80k_to_100k": 0.03372552360552188, "100k_to_200k": 0.0612588533574213, @@ -391,25 +685,25 @@ "25_to_34": { "household_weight": 20594001.0, "share_with_federal_education_debt": 0.3600484792377984, - "mean_among_borrowers": 35943.04970065601, + "mean_among_borrowers": 35938.930360187805, "median_among_borrowers": 23244.0 }, "35_to_49": { "household_weight": 34447794.528511114, "share_with_federal_education_debt": 0.2683793916575207, - "mean_among_borrowers": 41209.59995046775, + "mean_among_borrowers": 41207.86453627056, "median_among_borrowers": 23000.0 }, "50_to_61": { "household_weight": 26383416.705058806, "share_with_federal_education_debt": 0.14913648907893393, - "mean_among_borrowers": 41823.4843446509, + "mean_among_borrowers": 41820.081246818496, "median_among_borrowers": 28000.0 }, "62_and_older": { "household_weight": 47684957.45658214, "share_with_federal_education_debt": 0.03583557010066891, - "mean_among_borrowers": 51630.83962020747, + "mean_among_borrowers": 51625.44274182284, "median_among_borrowers": 36000.0 } }, @@ -417,25 +711,25 @@ "white_non_hispanic": { "household_weight": 81838505.71507175, "share_with_federal_education_debt": 0.1630534163575822, - "mean_among_borrowers": 38775.16233534788, + "mean_among_borrowers": 38771.91688875415, "median_among_borrowers": 23000.0 }, "black_non_hispanic": { "household_weight": 18706038.359267425, "share_with_federal_education_debt": 0.2289947160752199, - "mean_among_borrowers": 47894.9650347352, + "mean_among_borrowers": 47907.93891717693, "median_among_borrowers": 27000.0 }, "hispanic": { "household_weight": 15955872.224651583, "share_with_federal_education_debt": 0.1566171160511706, - "mean_among_borrowers": 33164.275274056155, + "mean_among_borrowers": 33168.61865028642, "median_among_borrowers": 20000.0 }, "other_or_multiple": { "household_weight": 18463274.3911613, "share_with_federal_education_debt": 0.1981094869447479, - "mean_among_borrowers": 35811.82171400663, + "mean_among_borrowers": 35786.5815808536, "median_among_borrowers": 20911.0 } }, @@ -443,71 +737,72 @@ "1": { "household_weight": 26990442.923403554, "share_with_federal_education_debt": 0.12126660088067846, - "mean_among_borrowers": 32701.99311366745, + "mean_among_borrowers": 32702.7958886962, "median_among_borrowers": 19200.0 }, "2": { "household_weight": 26994482.864333212, "share_with_federal_education_debt": 0.1559905239732057, - "mean_among_borrowers": 37718.60645255383, + "mean_among_borrowers": 37712.13786074628, "median_among_borrowers": 22000.0 }, "3": { "household_weight": 26997020.193277217, "share_with_federal_education_debt": 0.17955131715072067, - "mean_among_borrowers": 38482.880144490715, + "mean_among_borrowers": 38476.01343761174, "median_among_borrowers": 23000.0 }, "4": { "household_weight": 26985790.839470405, "share_with_federal_education_debt": 0.26346013869266394, - "mean_among_borrowers": 40527.082922078465, + "mean_among_borrowers": 40530.176936455646, "median_among_borrowers": 23195.0 }, "5": { "household_weight": 26995953.869667668, "share_with_federal_education_debt": 0.16089043903093728, - "mean_among_borrowers": 45104.97225220377, + "mean_among_borrowers": 45097.304974791696, "median_among_borrowers": 28000.0 } }, "borrowers": { - "borrower_weight": 25263299.640741862, - "mean_balance": 37067.473875380434, - "median_balance": 21634.0, + "borrower_weight": 25088578.38062776, + "mean_balance": 37322.86000407346, + "median_balance": 22000.0, "balance_bands": { - "under_5k": 0.11215076141602473, - "5k_to_10k": 0.16261828276540408, - "10k_to_20k": 0.1743648469362625, - "20k_to_40k": 0.22983573580807584, - "40k_to_60k": 0.12284267042874972, - "60k_to_80k": 0.0949401843921611, - "80k_to_100k": 0.029332915740791533, - "100k_to_200k": 0.057418087395041346, - "200k_plus": 0.016496515117489015 + "under_5k": 0.10677077633314812, + "5k_to_10k": 0.16632579551387022, + "10k_to_20k": 0.17454470788079757, + "20k_to_40k": 0.2286739571526593, + "40k_to_60k": 0.12048695309434337, + "60k_to_80k": 0.09765459818108711, + "80k_to_100k": 0.03345947236099234, + "100k_to_200k": 0.05493773460369269, + "200k_plus": 0.017146004879409302 }, "age_bands": { - "24_and_younger": 0.06351111547591387, - "25_to_34": 0.3387564753526982, - "35_to_49": 0.37092134292463996, - "50_to_61": 0.1558115646143939, - "62_and_older": 0.07099950163235395 - }, - "graduate_share_of_own_original_principal": 0.4607883219614359, - "ibr_new_borrower_share": 0.5830953867882087 + "24_and_younger": 0.11420848773987989, + "25_to_34": 0.3588039745460583, + "35_to_49": 0.3516343204975256, + "50_to_61": 0.11346958485303148, + "62_and_older": 0.061883632363504804 + }, + "graduate_share_of_own_original_principal": 0.49107247959697303, + "ibr_new_borrower_share": 0.570162419212365 } }, "metrics": { "share_with_debt_error_pp": -0.5321841415289907, - "mean_among_borrowers_error_pct": -4.092034203571204, + "mean_among_borrowers_error_pct": -4.099121538686434, "median_among_borrowers_error_pct": -7.9999999999999964, - "household_balance_band_tvd": 0.040650120081566585, + "household_balance_band_tvd": 0.040578309662763545, "head_age_band_share_mae_pp": 1.1934803926439914, "race_share_mae_pp": 1.077772642699106, "income_quintile_share_mae_pp": 0.8762098127830446, - "borrower_balance_band_tvd": 0.03696684344880254, - "borrower_age_band_tvd": 0.015639872914699815, - "household_rank_correlation": 0.17265294243545534 + "borrower_balance_band_tvd": 0.03713251302344053, + "borrower_age_band_tvd": 0.019972061845957664, + "household_rank_correlation": 0.17266749563083275, + "parent_plus_drawn_without_observed_parent_plus": 0.02526658514008899 } }, "shared_8_plus_household_income": { @@ -526,22 +821,22 @@ "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.17281627661600854, - "mean_among_borrowers": 39266.70030095491, + "mean_among_borrowers": 39254.97869162569, "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 915853474784.7556, - "parent_plus_share_of_total": 0.11558498718060221 + "federal_education_debt_total": 915580080877.1456, + "parent_plus_share_of_total": 0.059940399569770764 }, "household_balance_bands": { "under_5k": 0.08974701999643789, - "5k_to_10k": 0.1480250881106031, - "10k_to_20k": 0.1777595352503208, - "20k_to_40k": 0.24623523553505594, - "40k_to_60k": 0.11977492174284053, - "60k_to_80k": 0.10688238472563866, - "80k_to_100k": 0.03949620364967014, - "100k_to_200k": 0.051414568653002084, - "200k_plus": 0.020665042336430926 + "5k_to_10k": 0.14806900868476966, + "10k_to_20k": 0.1780192726055367, + "20k_to_40k": 0.2452616168183236, + "40k_to_60k": 0.1205246550102769, + "60k_to_80k": 0.10657649709866371, + "80k_to_100k": 0.039753518945480334, + "100k_to_200k": 0.05175325944273187, + "200k_plus": 0.020295151397779383 }, "by_head_age_band": { "24_and_younger": { @@ -553,25 +848,25 @@ "25_to_34": { "household_weight": 20594001.0, "share_with_federal_education_debt": 0.3474148475378485, - "mean_among_borrowers": 36843.74904984769, + "mean_among_borrowers": 36809.50492816768, "median_among_borrowers": 24600.0 }, "35_to_49": { "household_weight": 34447794.528511114, "share_with_federal_education_debt": 0.2717612000881406, - "mean_among_borrowers": 39300.240621441015, + "mean_among_borrowers": 39305.59411440302, "median_among_borrowers": 22000.0 }, "50_to_61": { "household_weight": 26383416.705058806, "share_with_federal_education_debt": 0.14495396516593925, - "mean_among_borrowers": 42167.25806737501, + "mean_among_borrowers": 42162.711454936776, "median_among_borrowers": 28000.0 }, "62_and_older": { "household_weight": 47684957.45658214, "share_with_federal_education_debt": 0.03258898091103795, - "mean_among_borrowers": 51808.82295387341, + "mean_among_borrowers": 51769.49362308479, "median_among_borrowers": 28000.0 } }, @@ -579,25 +874,25 @@ "white_non_hispanic": { "household_weight": 81838505.71507175, "share_with_federal_education_debt": 0.15882220643031406, - "mean_among_borrowers": 38343.12358920671, + "mean_among_borrowers": 38309.93027102956, "median_among_borrowers": 24000.0 }, "black_non_hispanic": { "household_weight": 18706038.359267425, "share_with_federal_education_debt": 0.224885334931214, - "mean_among_borrowers": 44936.30170394143, + "mean_among_borrowers": 44951.2117460371, "median_among_borrowers": 24600.0 }, "hispanic": { "household_weight": 15955872.224651583, "share_with_federal_education_debt": 0.16095386588360475, - "mean_among_borrowers": 34860.299992999004, + "mean_among_borrowers": 34893.011406958016, "median_among_borrowers": 20000.0 }, "other_or_multiple": { "household_weight": 18463274.3911613, "share_with_federal_education_debt": 0.1923427751654244, - "mean_among_borrowers": 39117.56739436241, + "mean_among_borrowers": 39120.753536492375, "median_among_borrowers": 22000.0 } }, @@ -605,71 +900,72 @@ "1": { "household_weight": 26990442.923403554, "share_with_federal_education_debt": 0.10260826272789049, - "mean_among_borrowers": 33872.567578817216, + "mean_among_borrowers": 33878.87020012928, "median_among_borrowers": 19200.0 }, "2": { "household_weight": 26994482.864333212, "share_with_federal_education_debt": 0.17100662350830068, - "mean_among_borrowers": 39205.32920511589, + "mean_among_borrowers": 39154.50811835669, "median_among_borrowers": 22000.0 }, "3": { "household_weight": 26997020.193277217, "share_with_federal_education_debt": 0.1785510426393907, - "mean_among_borrowers": 36610.58351254444, + "mean_among_borrowers": 36629.24348561517, "median_among_borrowers": 23000.0 }, "4": { "household_weight": 26985790.839470405, "share_with_federal_education_debt": 0.25540364615718963, - "mean_among_borrowers": 37762.92241632937, + "mean_among_borrowers": 37772.04606554361, "median_among_borrowers": 25000.0 }, "5": { "household_weight": 26995953.869667668, "share_with_federal_education_debt": 0.1565282419252913, - "mean_among_borrowers": 48351.71138032323, + "mean_among_borrowers": 48302.23313145786, "median_among_borrowers": 26000.0 } }, "borrowers": { - "borrower_weight": 24732420.497163415, - "mean_balance": 37030.48291976904, + "borrower_weight": 24592745.901716128, + "mean_balance": 37229.68083906623, "median_balance": 22000.0, "balance_bands": { - "under_5k": 0.10800838456724018, - "5k_to_10k": 0.15183570250699477, - "10k_to_20k": 0.18301823120692082, - "20k_to_40k": 0.2396488060359636, - "40k_to_60k": 0.1219331041101281, - "60k_to_80k": 0.09403649589832407, - "80k_to_100k": 0.03487135041034836, - "100k_to_200k": 0.04757724842740005, - "200k_plus": 0.01907067683668007 + "under_5k": 0.10360174630285794, + "5k_to_10k": 0.1504219956091371, + "10k_to_20k": 0.18311213927672274, + "20k_to_40k": 0.23971269075178334, + "40k_to_60k": 0.12279030310946719, + "60k_to_80k": 0.09669836723100285, + "80k_to_100k": 0.03608342021771613, + "100k_to_200k": 0.05000127236763974, + "200k_plus": 0.017578065133672955 }, "age_bands": { - "24_and_younger": 0.06096826553348787, - "25_to_34": 0.3331677198859893, - "35_to_49": 0.38398055689037613, - "50_to_61": 0.15779746384936916, - "62_and_older": 0.06408599384077752 - }, - "graduate_share_of_own_original_principal": 0.46198575187123647, - "ibr_new_borrower_share": 0.5919421297398388 + "24_and_younger": 0.10880795012149283, + "25_to_34": 0.35906309742386944, + "35_to_49": 0.36721906946956695, + "50_to_61": 0.10788831666800004, + "62_and_older": 0.057021566317070796 + }, + "graduate_share_of_own_original_principal": 0.4592646793399742, + "ibr_new_borrower_share": 0.5760474012257303 } }, "metrics": { "share_with_debt_error_pp": -0.8733291505995028, - "mean_among_borrowers_error_pct": -4.349507836301458, + "mean_among_borrowers_error_pct": -4.378060724440691, "median_among_borrowers_error_pct": -7.9999999999999964, - "household_balance_band_tvd": 0.037246827952770664, + "household_balance_band_tvd": 0.03627097953844585, "head_age_band_share_mae_pp": 1.350870233998188, "race_share_mae_pp": 1.3585851825102468, "income_quintile_share_mae_pp": 0.9228331015688831, - "borrower_balance_band_tvd": 0.030251768076485816, - "borrower_age_band_tvd": 0.008733492308356343, - "household_rank_correlation": 0.17515687252147844 + "borrower_balance_band_tvd": 0.032301626859559736, + "borrower_age_band_tvd": 0.008927557720090004, + "household_rank_correlation": 0.17512968016954317, + "parent_plus_drawn_without_observed_parent_plus": 0.02654694891508374 } }, "plus_household_income_and_education": { @@ -689,21 +985,21 @@ "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.1668063934073779, - "mean_among_borrowers": 38767.059113449155, + "mean_among_borrowers": 38760.6787211115, "median_among_borrowers": 23796.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 872755299812.5964, - "parent_plus_share_of_total": 0.11953256305130716 + "federal_education_debt_total": 872611659274.6003, + "parent_plus_share_of_total": 0.07414637298387526 }, "household_balance_bands": { "under_5k": 0.08808431892789516, "5k_to_10k": 0.13717658256161608, - "10k_to_20k": 0.18873503407584216, - "20k_to_40k": 0.24545468460672265, + "10k_to_20k": 0.18882635422463928, + "20k_to_40k": 0.2453633644579256, "40k_to_60k": 0.12322636723840236, "60k_to_80k": 0.10776956790540955, - "80k_to_100k": 0.03912114479997444, - "100k_to_200k": 0.055912505343134085, + "80k_to_100k": 0.03911980624752822, + "100k_to_200k": 0.05591384389558031, "200k_plus": 0.014519794541003451 }, "by_head_age_band": { @@ -716,25 +1012,25 @@ "25_to_34": { "household_weight": 20594001.0, "share_with_federal_education_debt": 0.35882688319565326, - "mean_among_borrowers": 36572.432216778245, + "mean_among_borrowers": 36572.66097268626, "median_among_borrowers": 24900.0 }, "35_to_49": { "household_weight": 34447794.528511114, "share_with_federal_education_debt": 0.25221217813313995, - "mean_among_borrowers": 38788.72407799315, + "mean_among_borrowers": 38787.71300020815, "median_among_borrowers": 22900.0 }, "50_to_61": { "household_weight": 26383416.705058806, "share_with_federal_education_debt": 0.13669823926255414, - "mean_among_borrowers": 40222.96469263708, + "mean_among_borrowers": 40230.335480571295, "median_among_borrowers": 25950.0 }, "62_and_older": { "household_weight": 47684957.45658214, "share_with_federal_education_debt": 0.03165003376804131, - "mean_among_borrowers": 50597.815332662714, + "mean_among_borrowers": 50489.727264864836, "median_among_borrowers": 37000.0 } }, @@ -742,25 +1038,25 @@ "white_non_hispanic": { "household_weight": 81838505.71507175, "share_with_federal_education_debt": 0.156024839562724, - "mean_among_borrowers": 38475.652655313614, - "median_among_borrowers": 23182.0 + "mean_among_borrowers": 38481.303452352964, + "median_among_borrowers": 23666.0 }, "black_non_hispanic": { "household_weight": 18706038.359267425, "share_with_federal_education_debt": 0.19389802688054777, - "mean_among_borrowers": 40781.818553326506, + "mean_among_borrowers": 40739.89648452815, "median_among_borrowers": 25166.0 }, "hispanic": { "household_weight": 15955872.224651583, "share_with_federal_education_debt": 0.15371847235012892, - "mean_among_borrowers": 32040.145053533837, + "mean_among_borrowers": 32038.772885804257, "median_among_borrowers": 20000.0 }, "other_or_multiple": { "household_weight": 18463274.3911613, "share_with_federal_education_debt": 0.1984583260312187, - "mean_among_borrowers": 42291.015044967404, + "mean_among_borrowers": 42274.537980522124, "median_among_borrowers": 25000.0 } }, @@ -774,65 +1070,66 @@ "2": { "household_weight": 26994482.864333212, "share_with_federal_education_debt": 0.16600786857086094, - "mean_among_borrowers": 39631.570218430665, + "mean_among_borrowers": 39624.46738359425, "median_among_borrowers": 21939.0 }, "3": { "household_weight": 26997020.193277217, "share_with_federal_education_debt": 0.1671000542140161, - "mean_among_borrowers": 34201.14487060498, + "mean_among_borrowers": 34192.31543281778, "median_among_borrowers": 21000.0 }, "4": { "household_weight": 26985790.839470405, "share_with_federal_education_debt": 0.25267861183865514, - "mean_among_borrowers": 34586.84037560038, + "mean_among_borrowers": 34599.69654081462, "median_among_borrowers": 25000.0 }, "5": { "household_weight": 26995953.869667668, "share_with_federal_education_debt": 0.15286484625519245, - "mean_among_borrowers": 46875.77881330891, + "mean_among_borrowers": 46837.093972152856, "median_among_borrowers": 28000.0 } }, "borrowers": { - "borrower_weight": 23668750.851507433, - "mean_balance": 36873.73724486232, + "borrower_weight": 23714170.340499215, + "mean_balance": 36797.056221880484, "median_balance": 23000.0, "balance_bands": { - "under_5k": 0.09598168882566091, - "5k_to_10k": 0.14287136790184726, - "10k_to_20k": 0.19074970787368226, - "20k_to_40k": 0.25231020514182567, - "40k_to_60k": 0.12373308878262614, - "60k_to_80k": 0.09562201893085862, - "80k_to_100k": 0.034750948986171766, - "100k_to_200k": 0.04987747161203311, - "200k_plus": 0.014103501945294196 + "under_5k": 0.09699471892619613, + "5k_to_10k": 0.1394638892424441, + "10k_to_20k": 0.18911050663109966, + "20k_to_40k": 0.2551113824446505, + "40k_to_60k": 0.12484286667522601, + "60k_to_80k": 0.09515433282795518, + "80k_to_100k": 0.03559212688986377, + "100k_to_200k": 0.049877581326660494, + "200k_plus": 0.013852595035904155 }, "age_bands": { - "24_and_younger": 0.057803501858546165, - "25_to_34": 0.35403082725820945, - "35_to_49": 0.36287233517496836, - "50_to_61": 0.1615925449287847, - "62_and_older": 0.06370079077949123 - }, - "graduate_share_of_own_original_principal": 0.4564621925903257, - "ibr_new_borrower_share": 0.5822440301014113 + "24_and_younger": 0.10242792997260068, + "25_to_34": 0.3760811152011804, + "35_to_49": 0.3477655420020849, + "50_to_61": 0.11573428915655427, + "62_and_older": 0.05799112366757967 + }, + "graduate_share_of_own_original_principal": 0.421014411591174, + "ibr_new_borrower_share": 0.564984415474899 } }, "metrics": { "share_with_debt_error_pp": -1.4743174714625662, - "mean_among_borrowers_error_pct": -5.566593181489421, + "mean_among_borrowers_error_pct": -5.582135298922686, "median_among_borrowers_error_pct": -4.815999999999998, - "household_balance_band_tvd": 0.04033667822097965, + "household_balance_band_tvd": 0.04033667822097967, "head_age_band_share_mae_pp": 1.3212277078973742, "race_share_mae_pp": 2.067159177073093, "income_quintile_share_mae_pp": 1.4743162952521935, - "borrower_balance_band_tvd": 0.05001011644613092, - "borrower_age_band_tvd": 0.023231363094113566, - "household_rank_correlation": 0.20794745847766363 + "borrower_balance_band_tvd": 0.052030960462776894, + "borrower_age_band_tvd": 0.030845057103948335, + "household_rank_correlation": 0.20793128913160808, + "parent_plus_drawn_without_observed_parent_plus": 0.01636273473995983 } }, "build_without_interest_and_ss_pension": { @@ -849,20 +1146,20 @@ "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.1785134113597766, - "mean_among_borrowers": 38129.7481875102, + "mean_among_borrowers": 38156.40076634909, "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 918653496595.8905, - "parent_plus_share_of_total": 0.12228737578012737 + "federal_education_debt_total": 919295632615.8619, + "parent_plus_share_of_total": 0.06816282381081624 }, "household_balance_bands": { - "under_5k": 0.09079966274592437, - "5k_to_10k": 0.1560993205378036, - "10k_to_20k": 0.17784594975720502, - "20k_to_40k": 0.24170300969285868, - "40k_to_60k": 0.12229052439445375, - "60k_to_80k": 0.0975892083544051, - "80k_to_100k": 0.045649331362645214, + "under_5k": 0.09126324438556496, + "5k_to_10k": 0.15516041383614745, + "10k_to_20k": 0.1786625820914109, + "20k_to_40k": 0.24094330640977704, + "40k_to_60k": 0.12235248869076151, + "60k_to_80k": 0.09731073610543892, + "80k_to_100k": 0.04628423532619501, "100k_to_200k": 0.04868017050483829, "200k_plus": 0.019342822649865955 }, @@ -876,51 +1173,51 @@ "25_to_34": { "household_weight": 20594001.0, "share_with_federal_education_debt": 0.36900446914360446, - "mean_among_borrowers": 35840.91733887638, + "mean_among_borrowers": 35924.35022877094, "median_among_borrowers": 23000.0 }, "35_to_49": { "household_weight": 34447794.528511114, "share_with_federal_education_debt": 0.2743786349921627, - "mean_among_borrowers": 38624.21584848466, - "median_among_borrowers": 22900.0 + "mean_among_borrowers": 38619.55355856089, + "median_among_borrowers": 22000.0 }, "50_to_61": { "household_weight": 26383416.705058806, "share_with_federal_education_debt": 0.14829262594950599, - "mean_among_borrowers": 40029.75297693061, + "mean_among_borrowers": 40031.502537207765, "median_among_borrowers": 27000.0 }, "62_and_older": { "household_weight": 47684957.45658214, "share_with_federal_education_debt": 0.03498656926046411, - "mean_among_borrowers": 48212.80171837554, - "median_among_borrowers": 26997.0 + "mean_among_borrowers": 48239.97130082438, + "median_among_borrowers": 27000.0 } }, "by_race": { "white_non_hispanic": { "household_weight": 81838505.71507175, "share_with_federal_education_debt": 0.1672189930591373, - "mean_among_borrowers": 36898.38867717323, + "mean_among_borrowers": 36915.04739897259, "median_among_borrowers": 23000.0 }, "black_non_hispanic": { "household_weight": 18706038.359267425, "share_with_federal_education_debt": 0.2324868349178142, - "mean_among_borrowers": 44763.01942694598, + "mean_among_borrowers": 44721.075062431184, "median_among_borrowers": 25000.0 }, "hispanic": { "household_weight": 15955872.224651583, "share_with_federal_education_debt": 0.16156695468894305, - "mean_among_borrowers": 35460.090234059295, + "mean_among_borrowers": 35711.49600365523, "median_among_borrowers": 20000.0 }, "other_or_multiple": { "household_weight": 18463274.3911613, "share_with_federal_education_debt": 0.18853789989668854, - "mean_among_borrowers": 36660.59407673795, + "mean_among_borrowers": 36645.789525487984, "median_among_borrowers": 21000.0 } }, @@ -928,71 +1225,72 @@ "1": { "household_weight": 26990442.923403554, "share_with_federal_education_debt": 0.10379714978765006, - "mean_among_borrowers": 34624.24650800998, + "mean_among_borrowers": 34621.48654269449, "median_among_borrowers": 19200.0 }, "2": { "household_weight": 26994482.864333212, "share_with_federal_education_debt": 0.17803919929996376, - "mean_among_borrowers": 37509.212621969265, + "mean_among_borrowers": 37503.22075890366, "median_among_borrowers": 21000.0 }, "3": { "household_weight": 26997020.193277217, "share_with_federal_education_debt": 0.17956752806930856, - "mean_among_borrowers": 35860.25899631823, + "mean_among_borrowers": 35880.44316170545, "median_among_borrowers": 21000.0 }, "4": { "household_weight": 26985790.839470405, "share_with_federal_education_debt": 0.2632482800176951, - "mean_among_borrowers": 38126.0224509717, + "mean_among_borrowers": 38136.3817053988, "median_among_borrowers": 25000.0 }, "5": { "household_weight": 26995953.869667668, "share_with_federal_education_debt": 0.16793147929877925, - "mean_among_borrowers": 43386.556076636596, + "mean_among_borrowers": 43498.44059935984, "median_among_borrowers": 28000.0 } }, "borrowers": { - "borrower_weight": 25709142.2609221, - "mean_balance": 35732.56109723443, + "borrower_weight": 25810010.594911456, + "mean_balance": 35617.79369424609, "median_balance": 21000.0, "balance_bands": { - "under_5k": 0.10244159338676109, - "5k_to_10k": 0.16637242925127055, - "10k_to_20k": 0.18690627049147995, - "20k_to_40k": 0.23582143486648904, - "40k_to_60k": 0.11958046299653502, - "60k_to_80k": 0.08637580964972973, - "80k_to_100k": 0.039420314567368823, - "100k_to_200k": 0.046956700106719984, - "200k_plus": 0.01612498468364574 + "under_5k": 0.10504821975483239, + "5k_to_10k": 0.1612084637239687, + "10k_to_20k": 0.1835278083390002, + "20k_to_40k": 0.2403341878392152, + "40k_to_60k": 0.1258185662605132, + "60k_to_80k": 0.08586155919483465, + "80k_to_100k": 0.03690144692037484, + "100k_to_200k": 0.04425676441601652, + "200k_plus": 0.01704298355124438 }, "age_bands": { - "24_and_younger": 0.0646454470512797, - "25_to_34": 0.3387609953816956, - "35_to_49": 0.37960479585619167, - "50_to_61": 0.15141879980849401, - "62_and_older": 0.06556996190233892 - }, - "graduate_share_of_own_original_principal": 0.4501205496319744, - "ibr_new_borrower_share": 0.5931987024130821 + "24_and_younger": 0.10884214784928073, + "25_to_34": 0.3590396433749492, + "35_to_49": 0.36321220703040163, + "50_to_61": 0.10958470645086948, + "62_and_older": 0.05932129529449908 + }, + "graduate_share_of_own_original_principal": 0.4472065188684173, + "ibr_new_borrower_share": 0.5842486917589438 } }, "metrics": { "share_with_debt_error_pp": -0.30361567622269603, - "mean_among_borrowers_error_pct": -7.119030826113226, + "mean_among_borrowers_error_pct": -7.054107309142554, "median_among_borrowers_error_pct": -7.9999999999999964, - "household_balance_band_tvd": 0.03158207267342461, + "household_balance_band_tvd": 0.030421622773926488, "head_age_band_share_mae_pp": 1.2596405967540016, "race_share_mae_pp": 1.069077118976515, "income_quintile_share_mae_pp": 0.6750791601254419, - "borrower_balance_band_tvd": 0.032948871945896414, - "borrower_age_band_tvd": 0.014803476410333236, - "household_rank_correlation": 0.16787602335617488 + "borrower_balance_band_tvd": 0.027269891261772868, + "borrower_age_band_tvd": 0.011203832648598016, + "household_rank_correlation": 0.1678412360138977, + "parent_plus_drawn_without_observed_parent_plus": 0.030096797445698646 } }, "build_plus_education": { @@ -1010,22 +1308,22 @@ "households": { "household_weight": 134963690.69015205, "share_with_federal_education_debt": 0.17234700775018835, - "mean_among_borrowers": 38142.78924085034, + "mean_among_borrowers": 38146.7986991102, "median_among_borrowers": 23000.0, "total_household_weight": 134963690.69015205, - "federal_education_debt_total": 887223715061.3356, - "parent_plus_share_of_total": 0.11798681584091122 + "federal_education_debt_total": 887316977419.0054, + "parent_plus_share_of_total": 0.0610214959298925 }, "household_balance_bands": { "under_5k": 0.094500931393709, "5k_to_10k": 0.1398997895474437, "10k_to_20k": 0.19368288257466282, "20k_to_40k": 0.23884340819629526, - "40k_to_60k": 0.1352262887879038, + "40k_to_60k": 0.13523397562363007, "60k_to_80k": 0.09436055969371929, "80k_to_100k": 0.03446366840523407, - "100k_to_200k": 0.0523437592826446, - "200k_plus": 0.0166787121183875 + "100k_to_200k": 0.052172845962700054, + "200k_plus": 0.01684193860260577 }, "by_head_age_band": { "24_and_younger": { @@ -1037,25 +1335,25 @@ "25_to_34": { "household_weight": 20594001.0, "share_with_federal_education_debt": 0.37451305639803345, - "mean_among_borrowers": 36295.97859113742, + "mean_among_borrowers": 36288.44136390066, "median_among_borrowers": 23900.0 }, "35_to_49": { "household_weight": 34447794.528511114, "share_with_federal_education_debt": 0.25660807838739297, - "mean_among_borrowers": 38679.04268701929, + "mean_among_borrowers": 38705.03441536813, "median_among_borrowers": 21600.0 }, "50_to_61": { "household_weight": 26383416.705058806, "share_with_federal_education_debt": 0.13992324697554848, - "mean_among_borrowers": 38624.20371347948, + "mean_among_borrowers": 38605.840676442705, "median_among_borrowers": 25000.0 }, "62_and_older": { "household_weight": 47684957.45658214, "share_with_federal_education_debt": 0.03311810732872758, - "mean_among_borrowers": 50709.27699844198, + "mean_among_borrowers": 50702.58315301913, "median_among_borrowers": 30000.0 } }, @@ -1063,25 +1361,25 @@ "white_non_hispanic": { "household_weight": 81838505.71507175, "share_with_federal_education_debt": 0.16168801258725676, - "mean_among_borrowers": 37236.74657199004, + "mean_among_borrowers": 37237.88141376128, "median_among_borrowers": 22900.0 }, "black_non_hispanic": { "household_weight": 18706038.359267425, "share_with_federal_education_debt": 0.20914133036844892, - "mean_among_borrowers": 44501.95583092358, + "mean_among_borrowers": 44561.07772792855, "median_among_borrowers": 26000.0 }, "hispanic": { "household_weight": 15955872.224651583, "share_with_federal_education_debt": 0.16318201246763214, - "mean_among_borrowers": 31794.600738997113, + "mean_among_borrowers": 31764.490083228888, "median_among_borrowers": 21000.0 }, "other_or_multiple": { "household_weight": 18463274.3911613, "share_with_federal_education_debt": 0.19023526055566922, - "mean_among_borrowers": 39178.989837148605, + "mean_among_borrowers": 39157.735848089564, "median_among_borrowers": 20000.0 } }, @@ -1089,13 +1387,13 @@ "1": { "household_weight": 26990442.923403554, "share_with_federal_education_debt": 0.10129901303895304, - "mean_among_borrowers": 42632.5471947818, + "mean_among_borrowers": 42632.530633246395, "median_among_borrowers": 20000.0 }, "2": { "household_weight": 26994482.864333212, "share_with_federal_education_debt": 0.17021674724220964, - "mean_among_borrowers": 39357.84941524378, + "mean_among_borrowers": 39357.02146920614, "median_among_borrowers": 20100.0 }, "3": { @@ -1107,53 +1405,54 @@ "4": { "household_weight": 26985790.839470405, "share_with_federal_education_debt": 0.2535480923849969, - "mean_among_borrowers": 35780.224907810196, + "mean_among_borrowers": 35764.93532409649, "median_among_borrowers": 25000.0 }, "5": { "household_weight": 26995953.869667668, "share_with_federal_education_debt": 0.16135595041628936, - "mean_among_borrowers": 44524.142074980824, + "mean_among_borrowers": 44570.452510837786, "median_among_borrowers": 28000.0 } }, "borrowers": { - "borrower_weight": 24672223.015597936, - "mean_balance": 35960.42863670725, - "median_balance": 20333.0, + "borrower_weight": 24629278.168691896, + "mean_balance": 36026.91769290015, + "median_balance": 20713.0, "balance_bands": { - "under_5k": 0.1094406791705482, - "5k_to_10k": 0.15004936475183045, - "10k_to_20k": 0.19555413431333177, - "20k_to_40k": 0.23372872974097667, - "40k_to_60k": 0.12914925746852132, - "60k_to_80k": 0.08602472899703802, - "80k_to_100k": 0.031224250567329498, - "100k_to_200k": 0.051343490636136885, - "200k_plus": 0.013485364354287193 + "under_5k": 0.11003236440751846, + "5k_to_10k": 0.1438146762421388, + "10k_to_20k": 0.1980177667743084, + "20k_to_40k": 0.2357539379328877, + "40k_to_60k": 0.13243383736799277, + "60k_to_80k": 0.08655648338436131, + "80k_to_100k": 0.030271579819877544, + "100k_to_200k": 0.048972446522624176, + "200k_plus": 0.014146907548290621 }, "age_bands": { - "24_and_younger": 0.06286336983939379, - "25_to_34": 0.3564829429265149, - "35_to_49": 0.36706810417884167, - "50_to_61": 0.14875323302634258, - "62_and_older": 0.06483235002890703 - }, - "graduate_share_of_own_original_principal": 0.4538844286126015, - "ibr_new_borrower_share": 0.5880863294842776 + "24_and_younger": 0.10435819675485504, + "25_to_34": 0.384565596966415, + "35_to_49": 0.34123822245830965, + "50_to_61": 0.10931470172073098, + "62_and_older": 0.060523282099689185 + }, + "graduate_share_of_own_original_principal": 0.4374325644888308, + "ibr_new_borrower_share": 0.5785380464302201 } }, "metrics": { "share_with_debt_error_pp": -0.9202560371815216, - "mean_among_borrowers_error_pct": -7.087263879546368, + "mean_among_borrowers_error_pct": -7.077497164927471, "median_among_borrowers_error_pct": -7.9999999999999964, - "household_balance_band_tvd": 0.02820605085678676, + "household_balance_band_tvd": 0.028213737692513032, "head_age_band_share_mae_pp": 1.7487117716451235, "race_share_mae_pp": 1.7889316725003699, "income_quintile_share_mae_pp": 0.920292137753204, - "borrower_balance_band_tvd": 0.029131316884503575, - "borrower_age_band_tvd": 0.030743346743266606, - "household_rank_correlation": 0.20717704021713643 + "borrower_balance_band_tvd": 0.034900124950217506, + "borrower_age_band_tvd": 0.037931773045254066, + "household_rank_correlation": 0.20717780916045597, + "parent_plus_drawn_without_observed_parent_plus": 0.02425600966419175 } } } @@ -1186,7 +1485,7 @@ "median_among_borrowers": 25000.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 785628046119.6727, - "parent_plus_share_of_total": 0.21495679635292875 + "parent_plus_share_of_total": 0.13018594263257258 }, "household_balance_bands": { "under_5k": 0.09597901426395242, @@ -1290,29 +1589,29 @@ } }, "borrowers": { - "borrower_weight": 18461703.3923496, - "mean_balance": 42554.4723270352, - "median_balance": 24000.0, + "borrower_weight": 18782588.57019364, + "mean_balance": 41827.46394000224, + "median_balance": 24700.0, "balance_bands": { - "under_5k": 0.1039144177980828, - "5k_to_10k": 0.16665917220324747, - "10k_to_20k": 0.15515901493993045, - "20k_to_40k": 0.23029463650667883, - "40k_to_60k": 0.10426754245310213, - "60k_to_80k": 0.09578243033185131, - "80k_to_100k": 0.05074417014558102, - "100k_to_200k": 0.06376833884555547, - "200k_plus": 0.029410276775970686 + "under_5k": 0.11010465011675885, + "5k_to_10k": 0.15766080070087685, + "10k_to_20k": 0.15145555688552018, + "20k_to_40k": 0.23876632487497312, + "40k_to_60k": 0.10328337810598322, + "60k_to_80k": 0.09709706504826032, + "80k_to_100k": 0.04973601963818461, + "100k_to_200k": 0.06276278750614343, + "200k_plus": 0.029133417123299275 }, "age_bands": { - "24_and_younger": 0.03769253987750687, - "25_to_34": 0.2662405413523508, - "35_to_49": 0.39817213354338704, - "50_to_61": 0.21071813444930676, - "62_and_older": 0.08717665077744856 - }, - "graduate_share_of_own_original_principal": 0.5360724443816975, - "ibr_new_borrower_share": 0.605734686749079 + "24_and_younger": 0.11708185382308234, + "25_to_34": 0.30330802369313054, + "35_to_49": 0.3722441005630567, + "50_to_61": 0.13909573807362918, + "62_and_older": 0.06827028384710118 + }, + "graduate_share_of_own_original_principal": 0.5079846787675676, + "ibr_new_borrower_share": 0.5790424319437123 } }, "metrics_vs_scf": { @@ -1323,8 +1622,8 @@ "head_age_band_share_mae_pp": 3.5543373606609885, "race_share_mae_pp": 2.8343426729118955, "income_quintile_share_mae_pp": 4.2522526924143715, - "borrower_balance_band_tvd": 0.06971797938468582, - "borrower_age_band_tvd": 0.08466988479278444 + "borrower_balance_band_tvd": 0.06560705296644279, + "borrower_age_band_tvd": 0.050081509772456775 } }, "scf_codings_all_candidates": { @@ -1354,7 +1653,7 @@ "median_among_borrowers": 25000.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 746948718213.9073, - "parent_plus_share_of_total": 0.15519605817424037 + "parent_plus_share_of_total": 0.10695080788545275 }, "household_balance_bands": { "under_5k": 0.10775015906187184, @@ -1458,29 +1757,29 @@ } }, "borrowers": { - "borrower_weight": 18531371.849430196, - "mean_balance": 40307.25432973677, + "borrower_weight": 18644433.43176055, + "mean_balance": 40062.82738211235, "median_balance": 24000.0, "balance_bands": { - "under_5k": 0.11848284445361304, - "5k_to_10k": 0.15391028318710992, - "10k_to_20k": 0.16124932842055162, - "20k_to_40k": 0.23782638366100617, - "40k_to_60k": 0.11453284638670623, - "60k_to_80k": 0.09065262966356538, - "80k_to_100k": 0.0390975463353386, - "100k_to_200k": 0.05475398408033307, - "200k_plus": 0.029494153811776037 + "under_5k": 0.11847637897193393, + "5k_to_10k": 0.15215298880238085, + "10k_to_20k": 0.15903005076767437, + "20k_to_40k": 0.24355676444848492, + "40k_to_60k": 0.11418906569388854, + "60k_to_80k": 0.09072876578657417, + "80k_to_100k": 0.035563245230000456, + "100k_to_200k": 0.05657395317657679, + "200k_plus": 0.02972878712248597 }, "age_bands": { - "24_and_younger": 0.05515156793888616, - "25_to_34": 0.3089032338721271, - "35_to_49": 0.3729683379284752, - "50_to_61": 0.19389979448680775, - "62_and_older": 0.06907706577370382 - }, - "graduate_share_of_own_original_principal": 0.4975010554094781, - "ibr_new_borrower_share": 0.6165370510095016 + "24_and_younger": 0.11887683513812723, + "25_to_34": 0.31829496102827626, + "35_to_49": 0.3727342252269422, + "50_to_61": 0.13256885684078035, + "62_and_older": 0.057525121765873946 + }, + "graduate_share_of_own_original_principal": 0.4878619324532848, + "ibr_new_borrower_share": 0.5971032517443657 } }, "metrics_vs_scf": { @@ -1491,8 +1790,8 @@ "head_age_band_share_mae_pp": 4.169585550417035, "race_share_mae_pp": 2.569878492307957, "income_quintile_share_mae_pp": 4.567500727054615, - "borrower_balance_band_tvd": 0.04019409665288553, - "borrower_age_band_tvd": 0.03235991026200852 + "borrower_balance_band_tvd": 0.04367455302813815, + "borrower_age_band_tvd": 0.035094572437311095 } }, "build": { @@ -1520,7 +1819,7 @@ "median_among_borrowers": 23000.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 858421208954.9309, - "parent_plus_share_of_total": 0.16758194869071996 + "parent_plus_share_of_total": 0.10691085971286082 }, "household_balance_bands": { "under_5k": 0.1179407532730876, @@ -1624,29 +1923,29 @@ } }, "borrowers": { - "borrower_weight": 21223403.433088087, - "mean_balance": 40446.915673129966, - "median_balance": 22000.0, + "borrower_weight": 21177503.938145176, + "mean_balance": 40534.57912047568, + "median_balance": 22224.0, "balance_bands": { - "under_5k": 0.12533388559616318, - "5k_to_10k": 0.13601201669515878, - "10k_to_20k": 0.18997031373843593, - "20k_to_40k": 0.2351308957545589, - "40k_to_60k": 0.10824433293789903, - "60k_to_80k": 0.08668533742450127, - "80k_to_100k": 0.03163070469899026, - "100k_to_200k": 0.05604778033785523, - "200k_plus": 0.03094473281643728 + "under_5k": 0.12755450607386423, + "5k_to_10k": 0.13657317586234458, + "10k_to_20k": 0.18059640738658841, + "20k_to_40k": 0.2390426854563625, + "40k_to_60k": 0.10720974798612737, + "60k_to_80k": 0.09004176213429058, + "80k_to_100k": 0.034401838885435285, + "100k_to_200k": 0.051526234839151065, + "200k_plus": 0.03305364137583606 }, "age_bands": { - "24_and_younger": 0.049988325664619956, - "25_to_34": 0.2917790224346921, - "35_to_49": 0.3780495021343625, - "50_to_61": 0.20778027670943267, - "62_and_older": 0.07240287305689257 - }, - "graduate_share_of_own_original_principal": 0.4826934460430418, - "ibr_new_borrower_share": 0.6147006160485154 + "24_and_younger": 0.12191368948550384, + "25_to_34": 0.3111513455503956, + "35_to_49": 0.37171601251334024, + "50_to_61": 0.13719732675785898, + "62_and_older": 0.05802162569290144 + }, + "graduate_share_of_own_original_principal": 0.4995084468534534, + "ibr_new_borrower_share": 0.6132625665271612 } }, "metrics_vs_scf": { @@ -1657,8 +1956,8 @@ "head_age_band_share_mae_pp": 2.3908244112012222, "race_share_mae_pp": 0.9415277758683394, "income_quintile_share_mae_pp": 2.865766910648814, - "borrower_balance_band_tvd": 0.04473225112436991, - "borrower_age_band_tvd": 0.049566199767822304 + "borrower_balance_band_tvd": 0.0457178698849765, + "borrower_age_band_tvd": 0.04223818791519181 } } }, @@ -1708,7 +2007,7 @@ "median_among_borrowers": 23000.0, "total_household_weight": 122537091.8856833, "federal_education_debt_total": 858421208954.9309, - "parent_plus_share_of_total": 0.16758194869071996 + "parent_plus_share_of_total": 0.10691085971286082 }, "household_balance_bands": { "under_5k": 0.1179407532730876, @@ -1812,29 +2111,29 @@ } }, "borrowers": { - "borrower_weight": 21223403.433088087, - "mean_balance": 40446.915673129966, - "median_balance": 22000.0, + "borrower_weight": 21177503.938145176, + "mean_balance": 40534.57912047568, + "median_balance": 22224.0, "balance_bands": { - "under_5k": 0.12533388559616318, - "5k_to_10k": 0.13601201669515878, - "10k_to_20k": 0.18997031373843593, - "20k_to_40k": 0.2351308957545589, - "40k_to_60k": 0.10824433293789903, - "60k_to_80k": 0.08668533742450127, - "80k_to_100k": 0.03163070469899026, - "100k_to_200k": 0.05604778033785523, - "200k_plus": 0.03094473281643728 + "under_5k": 0.12755450607386423, + "5k_to_10k": 0.13657317586234458, + "10k_to_20k": 0.18059640738658841, + "20k_to_40k": 0.2390426854563625, + "40k_to_60k": 0.10720974798612737, + "60k_to_80k": 0.09004176213429058, + "80k_to_100k": 0.034401838885435285, + "100k_to_200k": 0.051526234839151065, + "200k_plus": 0.03305364137583606 }, "age_bands": { - "24_and_younger": 0.049988325664619956, - "25_to_34": 0.2917790224346921, - "35_to_49": 0.3780495021343625, - "50_to_61": 0.20778027670943267, - "62_and_older": 0.07240287305689257 + "24_and_younger": 0.12191368948550384, + "25_to_34": 0.3111513455503956, + "35_to_49": 0.37171601251334024, + "50_to_61": 0.13719732675785898, + "62_and_older": 0.05802162569290144 }, - "graduate_share_of_own_original_principal": 0.4826934460430418, - "ibr_new_borrower_share": 0.6147006160485154 + "graduate_share_of_own_original_principal": 0.4995084468534534, + "ibr_new_borrower_share": 0.6132625665271612 } }, "metrics_vs_scf": { @@ -1845,8 +2144,8 @@ "head_age_band_share_mae_pp": 2.3908244112012222, "race_share_mae_pp": 0.9415277758683394, "income_quintile_share_mae_pp": 2.865766910648814, - "borrower_balance_band_tvd": 0.04473225112436991, - "borrower_age_band_tvd": 0.049566199767822304 + "borrower_balance_band_tvd": 0.0457178698849765, + "borrower_age_band_tvd": 0.04223818791519181 } } }