diff --git a/changelog.d/1073-scf-2025-education-debt.added.md b/changelog.d/1073-scf-2025-education-debt.added.md new file mode 100644 index 0000000000..4b32660999 --- /dev/null +++ b/changelog.d/1073-scf-2025-education-debt.added.md @@ -0,0 +1,11 @@ +The US build can impute federal student loan balances from the 2025 Survey of Consumer Finances (microcosm#1073, the student debt pilot). A new runtime module, `scf_education_debt`, writes four person inputs: `federal_student_loan_balance` and `federal_parent_plus_loan_balance` (policyengine-us#9721), and `is_income_based_repayment_new_borrower` and `federal_student_loan_graduate_share` (policyengine-us#9724). + +Only education debt reads the 2025 SCF. Net worth, the liquid-asset leaves and auto loans stay on the 2022 SCF. The `scf_wealth` stage declares the two 2025 files (`scfp2025s.zip`, `scf2025s.zip`), pinned by zip and member SHA-256 against a fresh Federal Reserve download, as artifacts 3 and 4, and a `scf_2025` vintage points at them. Moving the whole stage to 2025 later changes pinned coordinates, not code: the 2025 summary extract carries every column the shared predictor construction reads. + +The donor. Only loans flagged federal count. X7179, the total owed beyond the sixth loan, is spread over loans 1-6 in proportion to their balances, so it takes the family's federal share and holder split. Whose education decides the holder: the reference person's and the spouse's own loans go to `federal_student_loan_balance`, and codes 6 (multiple people), 7 (unrelated person) and -7 (other) go to the reference person's own balance. In families where the public file reversed respondent and spouse (X8000 = 1), the codes of loans 4 and 6 were left unreversed (none of 54 degree-discriminating loans match the coded person, against all 25 in other families), so the stage swaps them back. A loan for a child's education is the co-resident child's own loan when the household listing has a child aged 18 or over who could have been in school when it was borrowed, unless only a parent could have borrowed it: an undergraduate loan above the $31,000 limit on a dependent student's own Direct Loans, borrowed while the child could still have been dependent. Graduate loans and loans borrowed after the child turned 24 are never Parent PLUS. Every other loan for a child is the reference person's `federal_parent_plus_loan_balance`. This takes the Parent PLUS share of federal balances from 13.8% to 9.1% in the SCF and from 16.8% to 10.7% on the host, against 6.7% (a floor) in Federal Student Aid's portfolio, and the share of borrowers aged 24 or younger from 5.0% to 12.2% on the host (FSA: 13.8%). A person is an IBR new borrower when every federal loan they hold, Parent PLUS included, was borrowed in 2014 or later; that share is an upper bound, since paid-off pre-2014 loans are not observed and a consolidated loan is dated by its newest underlying loan. The graduate share weights each own loan's program by the amount borrowed. + +The model. One weighted QRF draws each household's federal education debt. A second, fit on borrower families and conditioning on that total and the number of adult children, draws the Parent PLUS, adult child's, reference person's and spouse's shares in that order, normalized to sum to one, then the graduate shares and new-borrower flags. Parent PLUS is drawn for itself, so a family like the donors without it rarely draws it. Both condition on the reference person's age, sex and race, a spouse or partner being present, the children of the reference person or partner, and the wages and income less capital gains of the reference person, spouse or partner and their children. The CPS side codes each predictor the way the SCF defines it: SCF `MARRIED` counts cohabiting partners, SCF `KIDS` counts the partner's children, the reference person comes from the householder couple (the man, or the older partner), and income covers the primary economic unit rather than the whole household. The shared interest-dividend and Social Security-pension predictors are left out (`SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS`). The CPS host carries tax-record imputations of both, so at reference ages 25-34 it reports them for 42% and 18% of households against 16% and 6% of SCF families, and that shift was the main reason the first host fit drew too few borrowers. The reference person, the member its `A_SPOUSE` (else `PECOHAB`) pointer names, and the youngest child of either aged 18 or over receive the balances; a child's draw in a household without an adult child is the reference person's Parent PLUS loan. A signal gate refuses negative, constant or incoherent columns and a borrower-household share outside 8-35%. + +The fiscal-refresh release tool runs the stage only with `--scf-education-debt`, because the pinned policyengine-us does not define the four inputs yet. The four columns join `US_STORED_NON_VARIABLE_COLUMNS` with the pending-engine reason the parent ids carry, and leave it when the engine pin moves. They are not in the release input-coverage manifest, whose check requires every column to be an engine input. No release is certified with them. + +Fit receipts are in `experiments/1073-scf-education-debt-receipts.md`, generated by `experiments/build_1073_scf_education_debt_receipt.py` on Modal through `experiments/modal_1073.py`, which clones a pushed commit, verifies every input's SHA-256 and records the run id in the receipt. The diagnosis behind the predictor and recipient choices is in `experiments/receipts/1073-predictor-shift-diagnosis.json` and `1073-recipient-definition-variants.json`. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 52fe0adb4f..f1c0c3ee7a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "e978ca7b15daf6eacf424fcb039f63c68b7eeabc1fb0c686b2715c75686b8c89", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32584, "claim_count": 49, "claims": [ { @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 316, + "pointer_sha256": "0212e52e2bf1e5996fe6b4785b2d74f2fb5680ec38b258c6ad4dbeadbd181a1b", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1716, - "pointer_sha256": "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + "pointer_count": 1884, + "pointer_sha256": "886eca845bf2e67788b19c28aa24459b9e3a5edb5f2d8129fb47d13804eca393", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -705,8 +705,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 112, - "pointer_sha256": "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", + "pointer_count": 120, + "pointer_sha256": "825b4ab09602353fc4d6c24b6f7654f7582502c7acf963b7ff6c117b07208eb4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1vintages.yaml", @@ -764,29 +764,29 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 63, - "pointer_sha256": "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", + "pointer_count": 67, + "pointer_sha256": "3f49902e745368ce8df0ad338a1697b76fd3299d35959977d36a8c09690ffb10", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/vintage_authorities", "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42368, + "consumed_field_count": 42368, "generation0_effect_counts": { - "legacy_behavior": 38498, - "no_generation0_effect": 3686 + "legacy_behavior": 38670, + "no_generation0_effect": 3698 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27729, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14010 + "legacy_behavior": 14182 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", - "resolved_binding_field_count": 9780, + "pointer_inventory_sha256": "f26245d12f4cd77a77ab4b595c9351f6923330815a87b5734e9c80e2a0a129a9", + "resolved_binding_field_count": 9784, "unused_field_count": 0 }, "inventory_coverage": { @@ -814,7 +814,7 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -824,7 +824,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", + "terminal_fields": 938 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", + "terminal_fields": 938 }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "429406ee38dbdcf5d751a86e37004636931b69f772e346fd8c853510ba54b05d" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "429406ee38dbdcf5d751a86e37004636931b69f772e346fd8c853510ba54b05d" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "ef7ed58e0170f21db8be7812f99251e702bb6b429aae463a80ea0a870c16ced5", "stages": 37 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "ef7ed58e0170f21db8be7812f99251e702bb6b429aae463a80ea0a870c16ced5", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "0af421c69e29c23eedd42b1d1e890b8223d637101b8b681e8fa0d07124591b72" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "0af421c69e29c23eedd42b1d1e890b8223d637101b8b681e8fa0d07124591b72" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "fec68ba41c34b4752e4bf7cbe31e35adcaa0e89be95ae522026a3a892883419e" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "19460fc168a6f9ed44e580d0e36777cce78bb33cb52b75630ee86203fa795684" + "spec_sha256": "eee022d2e258e9e7dfe6b87b80f4b731e3a679486a36abe016db5ec7515dfe10" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "19460fc168a6f9ed44e580d0e36777cce78bb33cb52b75630ee86203fa795684" + "spec_sha256": "eee022d2e258e9e7dfe6b87b80f4b731e3a679486a36abe016db5ec7515dfe10" }, "status": "pass" } diff --git a/experiments/1073-scf-education-debt-receipts.md b/experiments/1073-scf-education-debt-receipts.md new file mode 100644 index 0000000000..767713741f --- /dev/null +++ b/experiments/1073-scf-education-debt-receipts.md @@ -0,0 +1,200 @@ +# microcosm#1073 receipts: SCF 2025 federal student loan balances + +Generated 2026-10-23 by `experiments/build_1073_scf_education_debt_receipt.py` on branch `scf-2025-education-debt`, commit `60255dd9808ff1a0918e8c781b8818ce3376c32c`, Modal run `20261023T152015Z-60255dd98` (`experiments/modal_1073.py`). Machine-readable copy: `experiments/receipts/1073-scf-education-debt-fit.json`. + +Host columns impute the published default `populace-us-2024-spm-20260915` at its calibrated weights under three settings: the first receipt's construction (oldest adult, `A_MARITL` marriage, own children, whole-household income, every candidate predictor); the SCF-matching codings with every candidate predictor; and the build, which also leaves out interest and dividends and Social Security and pensions. The cause and the evidence are in the stage notes and `experiments/receipts/1073-predictor-shift-diagnosis.json`. + +Every figure is weighted. SCF rows pool the five implicates with `wgt` (X42001/5). Household figures group by the reference person's age and race and by weighted quintiles of income less capital gains of the setting's income unit, the income concept that setting conditions on. Borrower figures count people with a federal balance (own plus Parent PLUS), in Federal Student Aid's balance and age bands; SCF borrowers are the reference person, the spouse or partner and an adult child holding a loan for their own education, whose public-file age band (18-27 reads 25) is spread evenly over its years. Nothing here is a gate or a calibration target. + +## Source checks + +| Check | Loader | Fed Alternate Table 13 | +|---|---:|---:| +| Families with any education debt | 19.92% | 19.92% | +| Median education debt among them | $25,500 | $25,500 | + +| Share of education-loan balances | Value | +|---|---:| +| loans 1 6 of all education debt | 99.4% | +| x7179 remaining loans of all education debt | 0.6% | +| federal of loans 1 6 | 87.4% | +| non federal of loans 1 6 | 12.0% | +| federal status blank of loans 1 6 | 0.6% | + +| Federal balances on loans 1-6, by whose education (codes as published) | Share | +|---|---:| +| 1 respondent (reference person) | 54.8% | +| 2 spouse or partner | 31.4% | +| 3 child (incl. grandchild, other relative) | 13.8% | +| 6 multiple people | 0.0% | +| 7 unrelated person | 0.0% | +| -7 other | 0.0% | + +### X8000: are the whose-education codes reversed? + +Completed degree loans coded 1 or 2 in families with a spouse or partner. A loan discriminates when it is consistent with exactly one partner's highest degree (X5931, X6111: at least the program's level); the last columns count those consistent with the coded person (code 1 the reference person, code 2 the spouse or partner), as published, before the stage swaps codes 1 and 2 of loans 4 and 6 in reversed families. Counts pool the five implicates (implicate 1 in brackets). + +| Loans | X8000 | Code | Discriminating | Consistent with the coded person | +|---|---|---:|---:|---:| +| 1, 2, 3, 5 | not reversed | 1 | 180 (36) | 175 (35) | +| 1, 2, 3, 5 | not reversed | 2 | 208 (40) | 203 (40) | +| 1, 2, 3, 5 | reversed | 1 | 73 (15) | 70 (14) | +| 1, 2, 3, 5 | reversed | 2 | 470 (92) | 470 (92) | +| 4, 6 | not reversed | 1 | 20 (4) | 20 (4) | +| 4, 6 | not reversed | 2 | 5 (1) | 5 (1) | +| 4, 6 | reversed | 1 | 49 (9) | 0 (0) | +| 4, 6 | reversed | 2 | 5 (1) | 0 (0) | + +### Loans for a child's education (SCF 2025, weighted) + +FSA's Direct Parent PLUS balance is 6.7% of its portfolio at FY2026 Q3, a floor, since consolidated Parent PLUS loans sit in the consolidation total. + +| Reading | Parent PLUS share | Child's own share | Parent PLUS holders | Child borrowers | Borrowers | +|---|---:|---:|---:|---:|---:| +| Before: every code-3 loan Parent PLUS, codes as published | 13.8% | 0.0% | 4.4M | 0.0M | 27.2M | +| X8000 correction only | 13.8% | 0.0% | 4.4M | 0.0M | 26.9M | +| Build: co-resident adult child unless only a parent could borrow | 9.1% | 4.7% | 2.1M | 2.3M | 27.2M | +| Alternative: within-limit undergraduate loans for a dependent-age child also Parent PLUS | 11.8% | 2.0% | 4.1M | 0.4M | 27.1M | + +IBR new-borrower status is an upper bound: the SCF sees only loans still owed, so paid-off pre-2014 loans are missed, and a consolidated loan is dated by its most recent underlying loan. 29.3% of federal balances dated 2014 or later are on a loan that has been consolidated or refinanced (X9320-X9325). + +## Parent PLUS and borrowers against Federal Student Aid + +Before is the previous rule (every loan for a child's education the head's Parent PLUS loan, the residual of the composition draw); the host's before column is the build in the previous receipt (commit `2a6e65a73`, Modal run `20261022T153020Z-2a6e65a73`). FSA's Parent PLUS share is a floor; its dollars are the whole federal portfolio, of which household surveys capture part, so the shares compare and the levels do not. FSA borrowers come from its by-age table (FY2026 Q2), which counts the parent on a Parent PLUS loan; its 42.3M unduplicated recipients (FY2026 Q3) count the student instead. + +| | FSA | SCF 2025, before | SCF 2025 | Host: Build, before | Host: Build | +|---|---:|---:|---:|---:|---:| +| Parent PLUS share of federal balances | 6.7% (floor) | 13.8% | 9.1% | 16.8% | 10.7% | +| Parent PLUS balances | $115.0B | $138.3B | $91.2B | $143.9B | $91.8B | +| Federal balances | $1,724.5B | $1,005.9B | $1,005.9B | $858.4B | $858.4B | +| Borrowers (people) | 44.9M | 27.2M | 27.2M | 21.2M | 21.2M | +| Borrowers aged 24 and younger | 13.8% | 5.7% | 10.9% | 5.0% | 12.2% | +| Borrowers aged 25 to 34 | 31.2% | 33.1% | 35.3% | 29.2% | 31.1% | +| Borrowers aged 35 to 49 | 33.5% | 38.1% | 37.2% | 37.8% | 37.2% | +| Borrowers aged 50 to 61 | 14.3% | 16.2% | 11.2% | 20.8% | 13.7% | +| Borrowers aged 62 and older | 7.1% | 6.9% | 5.4% | 7.2% | 5.8% | + +## Fit + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| Households with federal education debt | 18.2% | 17.9% | 14.4% | 14.5% | 16.4% | +| Mean among them | $41,052 | $38,156 | $44,483 | $42,174 | $42,680 | +| Median among them | $25,000 | $23,000 | $25,000 | $25,000 | $23,000 | +| Weighted federal education debt | $1.006T | $0.919T | $0.786T | $0.747T | $0.858T | +| Parent PLUS share of the total | 9.1% | 6.8% | 13.0% | 10.7% | 10.7% | +| Borrowers (people) | 27.2M | 25.8M | 18.8M | 18.6M | 21.2M | +| Mean balance per borrower | $36,931 | $35,618 | $41,827 | $40,063 | $40,535 | +| Median balance per borrower | $21,000 | $21,000 | $24,700 | $24,000 | $22,224 | +| IBR new borrowers among borrowers (upper bound) | 59.5% | 58.4% | 57.9% | 59.7% | 61.3% | +| Graduate share of own original principal | 47.2% | 44.7% | 50.8% | 48.8% | 50.0% | + +### Borrowers by FSA balance band + +FSA: borrowers by debt size, FY2026 Q2. + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | FSA | +|---|---:|---:|---:|---:|---:|---:| +| under_5k | 11.9% | 10.5% | 11.0% | 11.8% | 12.8% | 15.4% | +| 5k_to_10k | 15.5% | 16.1% | 15.8% | 15.2% | 13.7% | 16.1% | +| 10k_to_20k | 18.2% | 18.4% | 15.1% | 15.9% | 18.1% | 20.5% | +| 20k_to_40k | 22.8% | 24.0% | 23.9% | 24.4% | 23.9% | 21.2% | +| 40k_to_60k | 13.0% | 12.6% | 10.3% | 11.4% | 10.7% | 9.4% | +| 60k_to_80k | 7.8% | 8.6% | 9.7% | 9.1% | 9.0% | 5.8% | +| 80k_to_100k | 3.7% | 3.7% | 5.0% | 3.6% | 3.4% | 3.1% | +| 100k_to_200k | 4.9% | 4.4% | 6.3% | 5.7% | 5.2% | 5.8% | +| 200k_plus | 2.2% | 1.7% | 2.9% | 3.0% | 3.3% | 2.7% | + +### Borrowers by FSA age band + +FSA: borrowers by age, FY2026 Q2, of those with an age reported. + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | FSA | +|---|---:|---:|---:|---:|---:|---:| +| 24_and_younger | 10.9% | 10.9% | 11.7% | 11.9% | 12.2% | 13.8% | +| 25_to_34 | 35.3% | 35.9% | 30.3% | 31.8% | 31.1% | 31.2% | +| 35_to_49 | 37.2% | 36.3% | 37.2% | 37.3% | 37.2% | 33.5% | +| 50_to_61 | 11.2% | 11.0% | 13.9% | 13.3% | 13.7% | 14.3% | +| 62_and_older | 5.4% | 5.9% | 6.8% | 5.8% | 5.8% | 7.1% | + +### Households with federal education debt, by age of reference person + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 24_and_younger | 22.2% | 25.0% | 22.7% | 21.1% | 21.8% | +| 25_to_34 | 36.2% | 36.9% | 26.7% | 27.1% | 28.1% | +| 35_to_49 | 27.2% | 27.4% | 22.3% | 20.9% | 24.4% | +| 50_to_61 | 16.7% | 14.8% | 14.2% | 13.3% | 16.4% | +| 62_and_older | 4.2% | 3.5% | 3.7% | 3.1% | 3.7% | + +Median among borrowers: + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 24_and_younger | $10,000 | $14,000 | $6,000 | $14,000 | $12,662 | +| 25_to_34 | $24,900 | $23,000 | $20,200 | $21,600 | $20,100 | +| 35_to_49 | $25,000 | $22,000 | $25,670 | $25,000 | $23,000 | +| 50_to_61 | $28,000 | $27,000 | $29,000 | $30,000 | $28,742 | +| 62_and_older | $26,000 | $27,000 | $38,000 | $26,000 | $34,000 | + +### Households with federal education debt, by race (`racecl4`) + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| white_non_hispanic | 17.2% | 16.7% | 13.3% | 13.3% | 15.3% | +| black_non_hispanic | 24.3% | 23.2% | 22.8% | 22.8% | 24.9% | +| hispanic | 14.1% | 16.2% | 12.3% | 11.8% | 13.8% | +| other_or_multiple | 19.5% | 18.9% | 15.3% | 17.0% | 18.4% | + +Median among borrowers: + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| white_non_hispanic | $23,000 | $23,000 | $25,000 | $25,000 | $23,000 | +| black_non_hispanic | $30,000 | $25,000 | $28,000 | $28,000 | $28,000 | +| hispanic | $23,000 | $20,000 | $24,764 | $23,000 | $19,200 | +| other_or_multiple | $20,000 | $21,000 | $24,000 | $22,000 | $18,000 | + +### Households with federal education debt, by income quintile + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 1 | 11.4% | 10.4% | 12.7% | 13.6% | 14.3% | +| 2 | 17.1% | 17.8% | 13.6% | 14.6% | 15.8% | +| 3 | 17.8% | 18.0% | 13.4% | 13.0% | 14.7% | +| 4 | 26.6% | 26.3% | 20.0% | 19.2% | 23.1% | +| 5 | 17.9% | 16.8% | 12.4% | 11.8% | 14.2% | + +Median among borrowers: + +| | SCF 2025 | SCF hold-out (build) | Host: First receipt | Host: SCF codings, all predictors | Host: Build | +|---|---:|---:|---:|---:|---:| +| 1 | $19,200 | $19,200 | $20,000 | $20,000 | $20,000 | +| 2 | $22,000 | $21,000 | $26,000 | $23,000 | $20,000 | +| 3 | $21,000 | $21,000 | $23,000 | $23,000 | $23,000 | +| 4 | $28,000 | $25,000 | $25,000 | $26,000 | $26,000 | +| 5 | $28,000 | $28,000 | $28,000 | $26,814 | $23,000 | + +## Predictor ablation (five-fold hold-out on SCF families) + +Each variant refits both QRFs. Errors are drawn minus observed; TVD is total variation distance between band distributions; MAE averages the absolute error of the share with debt across the groups. `educ` is the SCF reference person's education, which the CPS side cannot supply until microcosm#297, so that row shows what the predictor would add. The last column is the weighted share of held-out families observed with federal debt but no Parent PLUS that are drawn some Parent PLUS anyway. + +| Variant | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | household_rank_correlation | parent_plus_drawn_without_observed_parent_plus | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| shared_8_without_age | -0.250 | -1.707 | -4.000 | 0.023 | 4.812 | 0.648 | 0.672 | 0.033 | 0.122 | 0.154 | 0.046 | +| shared_8 | -0.532 | -4.099 | -8.000 | 0.041 | 1.193 | 1.078 | 0.876 | 0.037 | 0.020 | 0.173 | 0.025 | +| shared_8_plus_household_income | -0.873 | -4.378 | -8.000 | 0.036 | 1.351 | 1.359 | 0.923 | 0.032 | 0.009 | 0.175 | 0.027 | +| plus_household_income_and_education | -1.474 | -5.582 | -4.816 | 0.040 | 1.321 | 2.067 | 1.474 | 0.052 | 0.031 | 0.208 | 0.016 | +| build_without_interest_and_ss_pension (build) | -0.304 | -7.054 | -8.000 | 0.030 | 1.260 | 1.069 | 0.675 | 0.027 | 0.011 | 0.168 | 0.030 | +| build_plus_education | -0.920 | -7.077 | -8.000 | 0.028 | 1.749 | 1.789 | 0.920 | 0.035 | 0.038 | 0.207 | 0.024 | + +## Host fit, before and after + +Imputed with build seed 0 onto `populace-us-2024-spm-20260915` (57,240 households, sha256 `6496cc4393d4...`) at its calibrated household weights; the published weights were calibrated without these columns. Build signal gate: passed. Errors are imputed minus SCF 2025. + +| Setting | Households with debt | Parent PLUS share | share_with_debt_error_pp | mean_among_borrowers_error_pct | median_among_borrowers_error_pct | household_balance_band_tvd | head_age_band_share_mae_pp | race_share_mae_pp | income_quintile_share_mae_pp | borrower_balance_band_tvd | borrower_age_band_tvd | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| First receipt | 14.4% | 13.0% | -3.742 | 8.357 | 0.000 | 0.062 | 3.554 | 2.834 | 4.252 | 0.066 | 0.050 | +| SCF codings, all predictors | 14.5% | 10.7% | -3.701 | 2.732 | 0.000 | 0.040 | 4.170 | 2.570 | 4.568 | 0.044 | 0.035 | +| Build | 16.4% | 10.7% | -1.741 | 3.966 | -8.000 | 0.046 | 2.391 | 0.942 | 2.866 | 0.046 | 0.042 | +| SCF 2025 | 18.2% | 9.1% | | | | | | | | | | diff --git a/experiments/build_1073_scf_education_debt_receipt.py b/experiments/build_1073_scf_education_debt_receipt.py new file mode 100644 index 0000000000..6a2401b525 --- /dev/null +++ b/experiments/build_1073_scf_education_debt_receipt.py @@ -0,0 +1,1333 @@ +#!/usr/bin/env python +"""Fit receipts for the SCF 2025 federal student loan stage (microcosm#1073). + +Three measurements, all weighted, all against the 2025 SCF: + +1. **Source checks.** The loader's education debt against the Fed's published + Alternate Table 13 (19.92% of families hold education debt, median + $25,500), how the federal flag, X7179 and the whose-education codes split + the balances the X7179 and holder rules act on, the X8000 degree test for + loans 4 and 6, and the Parent PLUS share and borrowers under each reading + of a loan for a child's education. +2. **Hold-out fit and predictor ablation.** Families are split into five folds + by ``yy1`` (all implicates of a family stay together). Each fold is drawn + from the stage's two QRFs fit on the other four, and the drawn table is + compared with the observed one, for each predictor set. +3. **Host fit, before and after.** The stage imputed onto a US Microcosm H5 + (the published default unless ``--h5`` names another), tabulated at that + file's weights, under three settings (``HOST_SETTINGS``): the first receipt's + construction, the SCF-matching recipient definition with every candidate + predictor, and the build (that definition without the two shifted income + predictors). + +Federal Student Aid's portfolio tables (``FSA``) are the benchmark for the +Parent PLUS share, borrower counts and borrower bands. Nothing here is a gate +or a calibration target. The committed receipts are +produced on Modal by ``experiments/modal_1073.py`` (step ``receipt``), which +clones a pushed commit and verifies every input's SHA-256; the receipt records +that commit and the run id. A local run needs about 4 GB: + + .venv/Scripts/python.exe experiments/build_1073_scf_education_debt_receipt.py \ + --scf-zip-dir ~/.cache/scf --h5 populace_us_2024.h5 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from collections.abc import Mapping, Sequence +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pandas as pd + +import microcosm.build.us_runtime.scf_education_debt as education_debt +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + SCF_EDUCATION_DEBT_BALANCE_BANDS, + SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + SCF_EDUCATION_LOAN_YEAR_COLUMNS, + SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + SCF_REVERSED_COLUMN, + RecipientDefinition, + ScfPublicFile, + _read_stata_columns, + derive_scf_education_debt_targets, + draw_scf_education_debt_households, + fetch_scf_public_file, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, + with_us_scf_education_debt_inputs, +) +from microcosm.build.us_runtime.scf_wealth import SCF_WEALTH_PREDICTORS +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +ISSUE = "PolicyEngine/microcosm#1073" +FED_ALT_TABLE_13 = { + "source": ( + "Federal Reserve, 2025 SCF Chartbook tables, scf2025_tables_public_real_" + "historical.xlsx, sheet 'Table 13 25 Alt %s & median', education loans" + ), + "share_of_families_with_education_debt_pct": 19.92, + "median_among_holders": 25_500.0, +} +#: Federal Student Aid portfolio tables, as published (the chronicle#347 source +#: packages). Downloaded from +#: https://studentaid.gov/sites/default/files/fsawg/datacenter/library/; +#: the SHA-256 of each file read is listed. Recipients are students, so a +#: Parent PLUS loan counts the child; the by-age and by-debt-size tables count +#: borrowers, so they count the parent. +FSA = { + "files": { + "PortfolioSummary.xls": ( + "11606d4b93ceea969fcae0f790f3a4e1c2999b62050559ce89bdc269810a62ee" + ), + "PortfoliobyLoanType.xls": ( + "e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33" + ), + "Portfolio-by-Age.xls": ( + "c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef" + ), + "Portfolio-by-Debt-Size.xls": ( + "881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d" + ), + }, + "total_dollars_fy2026_q3": 1_724.5e9, + "unduplicated_recipients_fy2026_q3": 42.3e6, + "parent_plus_dollars_fy2026_q3": 115.0e9, + "parent_plus_recipients_fy2026_q3": 3.5e6, + "borrowers_by_age_fy2026_q2": { + "24_and_younger": 6.2e6, + "25_to_34": 14.0e6, + "35_to_49": 15.0e6, + "50_to_61": 6.4e6, + "62_and_older": 3.2e6, + "not_reported": 0.1e6, + }, + "borrowers_by_debt_size_fy2026_q2": { + "under_5k": 6.9e6, + "5k_to_10k": 7.2e6, + "10k_to_20k": 9.2e6, + "20k_to_40k": 9.5e6, + "40k_to_60k": 4.2e6, + "60k_to_80k": 2.6e6, + "80k_to_100k": 1.4e6, + "100k_to_200k": 2.6e6, + "200k_plus": 1.2e6, + }, +} +#: The build's host fit in the previous receipt (before loans for a child's +#: education were routed to co-resident children and Parent PLUS was drawn +#: for itself), for the before/after table. The SCF's own "before" column is +#: recomputed here from the old rule. +PREVIOUS_RECEIPT = { + "commit": "2a6e65a739b79a8232e1177610d11928fe92e26c", + "run": "20261022T153020Z-2a6e65a73", + "host_build": { + "parent_plus_share_of_total": 0.16758194869071996, + "federal_education_debt_total": 858421208954.9309, + "borrower_weight": 21223403.433088087, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257, + }, + }, +} +#: The published default the host fit imputes onto, and its release-manifest +#: SHA-256 (releases/populace-us-2024-spm-20260915/release_manifest.json). +DEFAULT_HOST_RELEASE = "populace-us-2024-spm-20260915" +DEFAULT_HOST_SHA256 = "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" +#: The build's predictor set, and the first receipt's (every candidate). +PRODUCTION_VARIANT = "build_without_interest_and_ss_pension" +FIRST_RECEIPT_VARIANT = "shared_8_plus_household_income" +VARIANTS: Mapping[str, tuple[str, ...]] = { + "shared_8_without_age": tuple(p for p in SCF_WEALTH_PREDICTORS if p != "age"), + "shared_8": SCF_WEALTH_PREDICTORS, + FIRST_RECEIPT_VARIANT: SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + "plus_household_income_and_education": ( + *SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + "educ", + ), + PRODUCTION_VARIANT: SCF_EDUCATION_DEBT_PREDICTORS, + "build_plus_education": (*SCF_EDUCATION_DEBT_PREDICTORS, "educ"), +} +#: Host settings for the before/after table: (label, definition, predictors). +BUILD_SETTING = "build" +HOST_SETTINGS: Mapping[str, tuple[str, RecipientDefinition, tuple[str, ...]]] = { + "first_receipt": ( + "First receipt", + SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + ), + "scf_codings_all_candidates": ( + "SCF codings, all predictors", + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + ), + BUILD_SETTING: ( + "Build", + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_PREDICTORS, + ), +} +WHOSE_LABELS = { + 1: "1 respondent (reference person)", + 2: "2 spouse or partner", + 3: "3 child (incl. grandchild, other relative)", + 6: "6 multiple people", + 7: "7 unrelated person", + -7: "-7 other", +} +_HOST_PERSON_COLUMNS = ( + "person_id", + "person_household_id", + "person_tax_unit_id", + "person_spm_unit_id", + "person_family_id", + "person_marital_unit_id", + "age", + "is_female", + "PRDTRACE", + "PRDTHSP", + "A_MARITL", + "A_LINENO", + "A_SPOUSE", + "PECOHAB", + "PH_SEQ", + "PEPAR1", + "PEPAR2", +) + + +def _git_commit() -> str: + import subprocess + + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=Path(__file__).resolve().parent, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _member_from_local_zip( + public_file: ScfPublicFile, zip_dir: Path, cache_dir: Path +) -> Path: + """Extract a pinned member from a locally cached Fed zip, verifying both pins.""" + + archive = zip_dir / public_file.zip_name + observed = _sha256(archive) + if observed != public_file.zip_sha256: + raise ValueError(f"{archive} sha-256 {observed} != {public_file.zip_sha256}") + cache_dir.mkdir(parents=True, exist_ok=True) + target = cache_dir / public_file.member + with zipfile.ZipFile(archive) as handle: + target.write_bytes(handle.read(public_file.member)) + observed = _sha256(target) + if observed != public_file.member_sha256: + raise ValueError(f"{target} sha-256 {observed} != {public_file.member_sha256}") + return target + + +def _pooled_median(values: np.ndarray, weights: np.ndarray) -> float: + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, cumulative[-1] / 2, side="left")) + return float(values[order][index]) + + +def source_checks(donor: pd.DataFrame, full: pd.DataFrame) -> dict[str, object]: + """The loader against the Fed's table, and the balances each rule acts on.""" + + weight = donor["scf_weight"].to_numpy(dtype=np.float64) + debt = donor["education_debt"].to_numpy(dtype=np.float64) + holders = debt > 0 + per_implicate = [ + 100.0 + * float(group.loc[group["education_debt"] > 0, "scf_weight"].sum()) + / float(group["scf_weight"].sum()) + for _, group in donor.groupby("implicate") + ] + full = full.reset_index(drop=True) + full_weight = full["wgt"].to_numpy(dtype=np.float64) + balances = np.column_stack( + [ + np.maximum(full[c].to_numpy(dtype=np.float64), 0.0) + for c in SCF_EDUCATION_LOAN_BALANCE_COLUMNS + ] + ) + federal = np.column_stack( + [full[c].to_numpy(dtype=np.float64) for c in SCF_EDUCATION_LOAN_FEDERAL_COLUMNS] + ) + whose = np.column_stack( + [full[c].to_numpy(dtype=np.float64) for c in SCF_EDUCATION_LOAN_WHOSE_COLUMNS] + ) + remaining = np.maximum( + full[SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN].to_numpy(dtype=np.float64), + 0.0, + ) + dollars = balances * full_weight[:, None] + loans_1_6 = float(dollars.sum()) + all_education = loans_1_6 + float(np.dot(remaining, full_weight)) + federal_dollars = np.where(federal == 1, dollars, 0.0) + by_whose = {} + for code, label in WHOSE_LABELS.items(): + by_whose[label] = float(federal_dollars[whose == code].sum()) / float( + federal_dollars.sum() + ) + other_codes = sorted( + {int(code) for code in np.unique(whose[balances > 0])} - set(WHOSE_LABELS) + ) + return { + "fed_alternate_table_13": FED_ALT_TABLE_13, + "loader_share_of_families_with_education_debt_pct": float( + np.mean(per_implicate) + ), + "loader_median_among_holders": _pooled_median(debt[holders], weight[holders]), + "families_per_implicate": int(len(donor) // 5), + "balance_shares": { + "loans_1_6_of_all_education_debt": loans_1_6 / all_education, + "x7179_remaining_loans_of_all_education_debt": 1.0 + - loans_1_6 / all_education, + "federal_of_loans_1_6": float(federal_dollars.sum()) / loans_1_6, + "non_federal_of_loans_1_6": float(np.where(federal == 5, dollars, 0).sum()) + / loans_1_6, + "federal_status_blank_of_loans_1_6": float( + np.where(federal == 0, dollars, 0).sum() + ) + / loans_1_6, + }, + "federal_balance_share_by_whose_education_code": by_whose, + "whose_education_codes_outside_the_rule": other_codes, + "federal_education_debt_after_x7179_rule": float( + np.dot(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], weight) + ), + "remaining_loans_without_detail_dollars": float( + np.dot(donor["remaining_loans_without_detail"], weight) + ), + } + + +_COMPLETED_COLUMNS = ("x7881", "x7886", "x7891", "x7896", "x7901", "x7996") +_CONSOLIDATED_COLUMNS = ("x9320", "x9321", "x9322", "x9323", "x9324", "x9325") +_DEGREE_LEVEL = {10: 1, 11: 1, 12: 2, 13: 3, 14: 4, 15: 4} +FULL_FILE_RECEIPT_COLUMNS = ( + "y1", + "yy1", + "x42001", + "x5931", + "x6111", + SCF_REVERSED_COLUMN, + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + *SCF_EDUCATION_LOAN_YEAR_COLUMNS, + *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + *_COMPLETED_COLUMNS, + *_CONSOLIDATED_COLUMNS, + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, +) + + +def _matrix(full: pd.DataFrame, columns: Sequence[str]) -> np.ndarray: + return np.column_stack( + [pd.to_numeric(full[c], errors="coerce").fillna(0).to_numpy() for c in columns] + ) + + +def x8000_degree_check(full: pd.DataFrame) -> list[dict[str, object]]: + """Do loans' whose-education codes match the coded person's degree? + + Completed loans coded 1 or 2 that financed a degree (programs 10-15), in + families with a spouse or partner (X6111 recorded). A loan is consistent + with a person whose highest degree (X5931 for the reference person, X6111 + for the spouse or partner) is at least the program's level; it + discriminates when it is consistent with exactly one of them. Raw codes, + before the stage's correction; counts pool the five implicates. + """ + + level = np.vectorize(lambda code: _DEGREE_LEVEL.get(int(code), 0)) + balance = _matrix(full, SCF_EDUCATION_LOAN_BALANCE_COLUMNS) + whose = _matrix(full, SCF_EDUCATION_LOAN_WHOSE_COLUMNS) + completed = _matrix(full, _COMPLETED_COLUMNS) + program_level = level(_matrix(full, SCF_EDUCATION_LOAN_PROGRAM_COLUMNS)) + reference = level(pd.to_numeric(full["x5931"]).fillna(0).to_numpy()) + spouse_degree = pd.to_numeric(full["x6111"]).fillna(0).to_numpy() + spouse = level(spouse_degree) + reversed_family = pd.to_numeric(full[SCF_REVERSED_COLUMN]).to_numpy() == 1 + implicate = pd.to_numeric(full["y1"]).astype(int).to_numpy() % 10 + rows = [] + for loans, positions in (("1, 2, 3, 5", (0, 1, 2, 4)), ("4, 6", (3, 5))): + for family, label in ((False, "not reversed"), (True, "reversed")): + for code in (1, 2): + counts = {"all": [0, 0], "implicate_1": [0, 0]} + for position in positions: + loan = ( + (balance[:, position] > 0) + & (whose[:, position] == code) + & (completed[:, position] == 1) + & (program_level[:, position] > 0) + & (spouse_degree != 0) + & (reversed_family == family) + ) + fits_reference = reference >= program_level[:, position] + fits_spouse = spouse >= program_level[:, position] + discriminating = loan & (fits_reference != fits_spouse) + coded = fits_reference if code == 1 else fits_spouse + for key, mask in ( + ("all", discriminating), + ("implicate_1", discriminating & (implicate == 1)), + ): + counts[key][0] += int(mask.sum()) + counts[key][1] += int((mask & coded).sum()) + rows.append( + { + "loans": loans, + "x8000": label, + "code": code, + "discriminating": counts["all"][0], + "consistent_with_coded_person": counts["all"][1], + "discriminating_implicate_1": counts["implicate_1"][0], + "consistent_implicate_1": counts["implicate_1"][1], + } + ) + return rows + + +def _derive_reading(full: pd.DataFrame, reading: str) -> pd.DataFrame: + """Donor targets under one reading of the whose-education codes. + + ``before`` is the previous rule: no X8000 correction and every loan for a + child's education the head's Parent PLUS loan (as if no child were + listed). ``x8000_only`` adds the correction. ``build`` is the stage. + ``within_limit_parent_plus`` reads an undergraduate loan within the + dependent limit for a child of dependent age as Parent PLUS too. + """ + + frame = full.copy() + if reading in {"before", "x8000_only"}: + for column in SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS: + frame[column] = 0 + if reading == "before": + frame[SCF_REVERSED_COLUMN] = 5 + if reading != "within_limit_parent_plus": + return derive_scf_education_debt_targets(frame) + limit = education_debt._DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT + education_debt._DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT = -1.0 + try: + return derive_scf_education_debt_targets(frame) + finally: + education_debt._DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT = limit + + +READINGS = ("before", "x8000_only", "build", "within_limit_parent_plus") + + +def child_education_readings(full: pd.DataFrame) -> dict[str, dict[str, float]]: + """Parent PLUS share and borrowers (people) under each reading.""" + + weight = full["wgt"].to_numpy(dtype=np.float64) + result = {} + for reading in READINGS: + targets = _derive_reading(full, reading) + total = float(np.dot(targets[SCF_EDUCATION_DEBT_TOTAL_TARGET], weight)) + plus = targets["federal_parent_plus_loan_balance"].to_numpy() + head = targets["head_federal_student_loan_balance"].to_numpy() + plus + spouse = targets["spouse_federal_student_loan_balance"].to_numpy() + child = targets["child_federal_student_loan_balance"].to_numpy() + result[reading] = { + "parent_plus_share_of_federal": float(np.dot(plus, weight)) / total, + "child_own_share_of_federal": float(np.dot(child, weight)) / total, + "parent_plus_dollars": float(np.dot(plus, weight)), + "borrowers": float( + weight[head > 0].sum() + + weight[spouse > 0].sum() + + weight[child > 0].sum() + ), + "parent_plus_holders": float(weight[plus > 0].sum()), + "child_borrowers": float(weight[child > 0].sum()), + } + return result + + +def consolidated_share_of_new_loans(full: pd.DataFrame) -> float: + """Share of federal balances dated 2014 or later on a consolidated loan.""" + + weight = full["wgt"].to_numpy(dtype=np.float64)[:, None] + balance = _matrix(full, SCF_EDUCATION_LOAN_BALANCE_COLUMNS) + federal = (_matrix(full, SCF_EDUCATION_LOAN_FEDERAL_COLUMNS) == 1) & (balance > 0) + new = _matrix(full, SCF_EDUCATION_LOAN_YEAR_COLUMNS) >= 2014 + consolidated = _matrix(full, _CONSOLIDATED_COLUMNS) == 1 + dated_new = np.where(federal & new, balance, 0.0) * weight + return float(np.where(consolidated, dated_new, 0.0).sum() / dated_new.sum()) + + +def donor_under_reading( + donor: pd.DataFrame, full: pd.DataFrame, reading: str +) -> pd.DataFrame: + """The donor with its targets derived under another reading.""" + + targets = _derive_reading(full, reading) + keys = pd.MultiIndex.from_arrays( + [ + pd.to_numeric(full["yy1"]).to_numpy(), + pd.to_numeric(full["y1"]).astype(int).to_numpy() % 10, + ] + ) + targets.index = keys + aligned = targets.reindex( + pd.MultiIndex.from_arrays( + [donor["yy1"].to_numpy(), donor["implicate"].to_numpy()] + ) + ) + result = donor.copy() + for column in targets.columns: + result[column] = aligned[column].to_numpy() + return result + + +def parent_plus_draw_rate(donor: pd.DataFrame, drawn: pd.DataFrame) -> float: + """Weighted share drawing Parent PLUS among held-out borrower families without it. + + Families observed with federal education debt but no Parent PLUS balance, + and drawn with debt: how often the draw gives them Parent PLUS anyway. + """ + + weight = donor["scf_weight"].to_numpy(dtype=np.float64) + observed_without = (donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0) & ( + donor["federal_parent_plus_loan_balance"] <= 0 + ) + drawn_debt = drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0 + base = (observed_without & drawn_debt).to_numpy() + plus = (drawn["federal_parent_plus_loan_balance"] > 0).to_numpy() + return float(weight[base & plus].sum() / weight[base].sum()) + + +def _fold_of_family(families: np.ndarray, *, folds: int, seed: int) -> np.ndarray: + unique = np.unique(families) + rng = np.random.default_rng(np.random.SeedSequence([seed, 1073, 5])) + assignment = pd.Series(rng.permutation(len(unique)) % folds, index=unique) + return assignment.reindex(families).to_numpy() + + +def holdout_draws( + donor: pd.DataFrame, + predictors: Sequence[str], + *, + folds: int, + seed: int, + n_estimators: int, +) -> pd.DataFrame: + """Draw every SCF row from models fit on the other folds' families.""" + + fold = _fold_of_family(donor["yy1"].to_numpy(), folds=folds, seed=seed) + parts = [] + for k in range(folds): + held_out = fold == k + drawn = draw_scf_education_debt_households( + donor.loc[ + held_out, [*predictors, SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] + ], + donor.loc[~held_out], + seed=seed + k, + predictors=predictors, + n_estimators=n_estimators, + ) + parts.append(drawn) + return pd.concat(parts).reindex(donor.index) + + +def _flatten(table: Mapping[str, object], prefix: str = "") -> dict[str, float]: + flat: dict[str, float] = {} + for key, value in table.items(): + name = f"{prefix}{key}" + if isinstance(value, Mapping): + flat.update(_flatten(value, prefix=f"{name}.")) + elif isinstance(value, (int, float)): + flat[name] = float(value) + return flat + + +def fit_metrics( + observed: Mapping[str, object], drawn: Mapping[str, object] +) -> dict[str, float]: + """Distributional distances between two fit tables (pp = percentage points).""" + + def share_mae(section: str) -> float: + groups = observed[section] + return float( + np.nanmean( + [ + abs( + drawn[section][g]["share_with_federal_education_debt"] + - groups[g]["share_with_federal_education_debt"] + ) + for g in groups + ] + ) + * 100 + ) + + def tvd(a: Mapping[str, float], b: Mapping[str, float]) -> float: + return 0.5 * float(sum(abs(a[k] - b[k]) for k in a)) + + households_o = observed["households"] + households_d = drawn["households"] + return { + "share_with_debt_error_pp": 100 + * ( + households_d["share_with_federal_education_debt"] + - households_o["share_with_federal_education_debt"] + ), + "mean_among_borrowers_error_pct": 100 + * ( + households_d["mean_among_borrowers"] / households_o["mean_among_borrowers"] + - 1 + ), + "median_among_borrowers_error_pct": 100 + * ( + households_d["median_among_borrowers"] + / households_o["median_among_borrowers"] + - 1 + ), + "household_balance_band_tvd": tvd( + observed["household_balance_bands"], drawn["household_balance_bands"] + ), + "head_age_band_share_mae_pp": share_mae("by_head_age_band"), + "race_share_mae_pp": share_mae("by_race"), + "income_quintile_share_mae_pp": share_mae("by_income_quintile"), + "borrower_balance_band_tvd": tvd( + observed["borrowers"]["balance_bands"], drawn["borrowers"]["balance_bands"] + ), + "borrower_age_band_tvd": tvd( + observed["borrowers"]["age_bands"], drawn["borrowers"]["age_bands"] + ), + } + + +def _rank_correlation(a: np.ndarray, b: np.ndarray) -> float: + ranks_a = pd.Series(a).rank().to_numpy() + ranks_b = pd.Series(b).rank().to_numpy() + return float(np.corrcoef(ranks_a, ranks_b)[0, 1]) + + +def load_host_frame(path: Path) -> Frame: + """Read the columns the stage needs from a Microcosm US H5 (table format).""" + + from microcosm.build.us_runtime.scf_education_debt import ( + _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS, + ) + + with pd.HDFStore(path, "r") as store: + available = set(store.get_storer("/person").data_columns) + columns = [c for c in _HOST_PERSON_COLUMNS if c in available] + [ + c for c in _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS if c in available + ] + person = store.select("/person", columns=columns) + household = store.select( + "/household", columns=["household_id", "household_weight"] + ) + tables = { + "person": person.reset_index(drop=True), + "household": household.loc[:, ["household_id"]].reset_index(drop=True), + } + for entity in ("tax_unit", "spm_unit", "family", "marital_unit"): + tables[entity] = store.select( + f"/{entity}", columns=[f"{entity}_id"] + ).reset_index(drop=True) + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + values=household["household_weight"].to_numpy(dtype=np.float64), + kind=WeightKind.CALIBRATED, + ) + }, + ) + + +def _pct(value: float) -> str: + return "" if value != value else f"{100 * value:.1f}%" + + +def _dollars(value: float) -> str: + return "" if value != value else f"${value:,.0f}" + + +def markdown(receipt: Mapping[str, object]) -> str: + """Render the receipt's tables as the committed Markdown note.""" + + observed = receipt["scf_2025_observed"] + holdout = receipt["holdout"]["variants"][PRODUCTION_VARIANT]["fit"] + host = receipt.get("host", {}).get("fit") + settings = receipt.get("host_settings", {}) + columns = [("SCF 2025", observed), ("SCF hold-out (build)", holdout)] + for setting in settings.values(): + columns.append((f"Host: {setting['label']}", setting["fit"])) + header = "| | " + " | ".join(label for label, _ in columns) + " |" + rule = "|---|" + "---:|" * len(columns) + + def row(label: str, getter, formatter) -> str: + return ( + f"| {label} | " + + " | ".join(formatter(getter(table)) for _, table in columns) + + " |" + ) + + lines = [ + "# microcosm#1073 receipts: SCF 2025 federal student loan balances", + "", + f"Generated {receipt['generated_at']} by " + "`experiments/build_1073_scf_education_debt_receipt.py` on branch " + f"`scf-2025-education-debt`, commit `{receipt['provenance']['commit']}`" + + ( + f", Modal run `{receipt['provenance']['run']}` " + "(`experiments/modal_1073.py`)" + if receipt["provenance"].get("run") + else "" + ) + + ". Machine-readable copy: " + "`experiments/receipts/1073-scf-education-debt-fit.json`.", + "", + "Host columns impute the published default " + f"`{receipt.get('host', {}).get('release', DEFAULT_HOST_RELEASE)}` at its " + "calibrated weights under three settings: the first receipt's construction " + "(oldest adult, `A_MARITL` marriage, own children, whole-household income, " + "every candidate predictor); the SCF-matching codings with every candidate " + "predictor; and the build, which also leaves out interest and dividends and " + "Social Security and pensions. The cause and the evidence are in the " + "stage notes and `experiments/receipts/1073-predictor-shift-diagnosis.json`.", + "", + "Every figure is weighted. SCF rows pool the five implicates with `wgt` " + "(X42001/5). Household figures group by the reference person's age and " + "race and by weighted quintiles of income less capital gains of the " + "setting's income unit, the income concept that setting conditions on. " + "Borrower figures count " + "people with a federal balance (own plus Parent PLUS), in Federal Student " + "Aid's balance and age bands; SCF borrowers are the reference person, the " + "spouse or partner and an adult child holding a loan for their own " + "education, whose public-file age band (18-27 reads 25) is spread evenly " + "over its years. Nothing here is a gate or a calibration target.", + "", + "## Source checks", + "", + ] + checks = receipt["source_checks"] + lines += [ + "| Check | Loader | Fed Alternate Table 13 |", + "|---|---:|---:|", + "| Families with any education debt | " + f"{checks['loader_share_of_families_with_education_debt_pct']:.2f}% | " + f"{FED_ALT_TABLE_13['share_of_families_with_education_debt_pct']:.2f}% |", + "| Median education debt among them | " + f"{_dollars(checks['loader_median_among_holders'])} | " + f"{_dollars(FED_ALT_TABLE_13['median_among_holders'])} |", + "", + "| Share of education-loan balances | Value |", + "|---|---:|", + ] + for key, value in checks["balance_shares"].items(): + lines.append(f"| {key.replace('_', ' ')} | {_pct(value)} |") + lines += [ + "", + "| Federal balances on loans 1-6, by whose education (codes as published) " + "| Share |", + "|---|---:|", + ] + for key, value in checks["federal_balance_share_by_whose_education_code"].items(): + lines.append(f"| {key} | {_pct(value)} |") + lines += [ + "", + "### X8000: are the whose-education codes reversed?", + "", + "Completed degree loans coded 1 or 2 in families with a spouse or partner. " + "A loan discriminates when it is consistent with exactly one partner's " + "highest degree (X5931, X6111: at least the program's level); the last " + "columns count those consistent with the coded person (code 1 the " + "reference person, code 2 the spouse or partner), as published, before " + "the stage swaps codes 1 and 2 of loans 4 and 6 in reversed families. " + "Counts pool the five implicates (implicate 1 in brackets).", + "", + "| Loans | X8000 | Code | Discriminating | Consistent with the coded person |", + "|---|---|---:|---:|---:|", + ] + for entry in checks["x8000_degree_check"]: + lines.append( + f"| {entry['loans']} | {entry['x8000']} | {entry['code']} | " + f"{entry['discriminating']} ({entry['discriminating_implicate_1']}) | " + f"{entry['consistent_with_coded_person']} " + f"({entry['consistent_implicate_1']}) |" + ) + readings = checks["child_education_readings"] + reading_labels = { + "before": "Before: every code-3 loan Parent PLUS, codes as published", + "x8000_only": "X8000 correction only", + "build": "Build: co-resident adult child unless only a parent could borrow", + "within_limit_parent_plus": ( + "Alternative: within-limit undergraduate loans for a dependent-age " + "child also Parent PLUS" + ), + } + lines += [ + "", + "### Loans for a child's education (SCF 2025, weighted)", + "", + "FSA's Direct Parent PLUS balance is " + f"{_pct(FSA['parent_plus_dollars_fy2026_q3'] / FSA['total_dollars_fy2026_q3'])}" + " of its portfolio at FY2026 Q3, a floor, since consolidated Parent PLUS " + "loans sit in the consolidation total.", + "", + "| Reading | Parent PLUS share | Child's own share | Parent PLUS holders " + "| Child borrowers | Borrowers |", + "|---|---:|---:|---:|---:|---:|", + ] + for reading, label in reading_labels.items(): + entry = readings[reading] + lines.append( + f"| {label} | {_pct(entry['parent_plus_share_of_federal'])} | " + f"{_pct(entry['child_own_share_of_federal'])} | " + f"{entry['parent_plus_holders'] / 1e6:,.1f}M | " + f"{entry['child_borrowers'] / 1e6:,.1f}M | " + f"{entry['borrowers'] / 1e6:,.1f}M |" + ) + lines += [ + "", + "IBR new-borrower status is an upper bound: the SCF sees only loans still " + "owed, so paid-off pre-2014 loans are missed, and a consolidated loan is " + "dated by its most recent underlying loan. " + f"{_pct(checks['consolidated_share_of_federal_balances_dated_2014_or_later'])}" + " of federal balances dated 2014 or later are on a loan that has been " + "consolidated or refinanced (X9320-X9325).", + ] + before = receipt["scf_2025_observed_before"] + previous = receipt["previous_receipt"]["host_build"] + build = settings.get(BUILD_SETTING, {}).get("fit") + by_age_fsa = { + k: v + for k, v in FSA["borrowers_by_age_fy2026_q2"].items() + if k != "not_reported" + } + fsa_age_total = sum(by_age_fsa.values()) + fsa_borrowers = sum(FSA["borrowers_by_age_fy2026_q2"].values()) + compare = [ + ("SCF 2025, before", { + "pp": before["households"]["parent_plus_share_of_total"], + "pp_dollars": before["households"]["parent_plus_share_of_total"] + * before["households"]["federal_education_debt_total"], + "total": before["households"]["federal_education_debt_total"], + "borrowers": before["borrowers"]["borrower_weight"], + "ages": before["borrowers"]["age_bands"], + }), + ("SCF 2025", { + "pp": observed["households"]["parent_plus_share_of_total"], + "pp_dollars": observed["households"]["parent_plus_share_of_total"] + * observed["households"]["federal_education_debt_total"], + "total": observed["households"]["federal_education_debt_total"], + "borrowers": observed["borrowers"]["borrower_weight"], + "ages": observed["borrowers"]["age_bands"], + }), + ("Host: Build, before", { + "pp": previous["parent_plus_share_of_total"], + "pp_dollars": previous["parent_plus_share_of_total"] + * previous["federal_education_debt_total"], + "total": previous["federal_education_debt_total"], + "borrowers": previous["borrower_weight"], + "ages": previous["age_bands"], + }), + ] # fmt: skip + if build is not None: + compare.append( + ( + "Host: Build", + { + "pp": build["households"]["parent_plus_share_of_total"], + "pp_dollars": build["households"]["parent_plus_share_of_total"] + * build["households"]["federal_education_debt_total"], + "total": build["households"]["federal_education_debt_total"], + "borrowers": build["borrowers"]["borrower_weight"], + "ages": build["borrowers"]["age_bands"], + }, + ) + ) + lines += [ + "", + "## Parent PLUS and borrowers against Federal Student Aid", + "", + "Before is the previous rule (every loan for a child's education the " + "head's Parent PLUS loan, the residual of the composition draw); the " + "host's before column is the build in the previous receipt (commit " + f"`{receipt['previous_receipt']['commit'][:9]}`, Modal run " + f"`{receipt['previous_receipt']['run']}`). FSA's Parent PLUS share is a " + "floor; its dollars are the whole federal portfolio, of which household " + "surveys capture part, so the shares compare and the levels do not. FSA " + "borrowers come from its by-age table (FY2026 Q2), which counts the " + "parent on a Parent PLUS loan; its " + f"{FSA['unduplicated_recipients_fy2026_q3'] / 1e6:.1f}M unduplicated " + "recipients (FY2026 Q3) count the student instead.", + "", + "| | FSA | " + " | ".join(label for label, _ in compare) + " |", + "|---|---:|" + "---:|" * len(compare), + "| Parent PLUS share of federal balances | " + f"{_pct(FSA['parent_plus_dollars_fy2026_q3'] / FSA['total_dollars_fy2026_q3'])}" + " (floor) | " + " | ".join(_pct(c["pp"]) for _, c in compare) + " |", + f"| Parent PLUS balances | ${FSA['parent_plus_dollars_fy2026_q3'] / 1e9:,.1f}B | " + + " | ".join(f"${c['pp_dollars'] / 1e9:,.1f}B" for _, c in compare) + + " |", + f"| Federal balances | ${FSA['total_dollars_fy2026_q3'] / 1e9:,.1f}B | " + + " | ".join(f"${c['total'] / 1e9:,.1f}B" for _, c in compare) + + " |", + f"| Borrowers (people) | {fsa_borrowers / 1e6:,.1f}M | " + + " | ".join(f"{c['borrowers'] / 1e6:,.1f}M" for _, c in compare) + + " |", + ] + for label, _, _ in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: + lines.append( + f"| Borrowers aged {label.replace('_', ' ')} | " + f"{_pct(by_age_fsa[label] / fsa_age_total)} | " + + " | ".join(_pct(c["ages"][label]) for _, c in compare) + + " |" + ) + lines += [ + "", + "## Fit", + "", + header, + rule, + row( + "Households with federal education debt", + lambda t: t["households"]["share_with_federal_education_debt"], + _pct, + ), + row( + "Mean among them", + lambda t: t["households"]["mean_among_borrowers"], + _dollars, + ), + row( + "Median among them", + lambda t: t["households"]["median_among_borrowers"], + _dollars, + ), + row( + "Weighted federal education debt", + lambda t: t["households"]["federal_education_debt_total"], + lambda v: f"${v / 1e12:,.3f}T", + ), + row( + "Parent PLUS share of the total", + lambda t: t["households"]["parent_plus_share_of_total"], + _pct, + ), + row( + "Borrowers (people)", + lambda t: t["borrowers"]["borrower_weight"], + lambda v: f"{v / 1e6:,.1f}M", + ), + row( + "Mean balance per borrower", + lambda t: t["borrowers"]["mean_balance"], + _dollars, + ), + row( + "Median balance per borrower", + lambda t: t["borrowers"]["median_balance"], + _dollars, + ), + row( + "IBR new borrowers among borrowers (upper bound)", + lambda t: t["borrowers"]["ibr_new_borrower_share"], + _pct, + ), + row( + "Graduate share of own original principal", + lambda t: t["borrowers"]["graduate_share_of_own_original_principal"], + _pct, + ), + "", + "### Borrowers by FSA balance band", + "", + "FSA: borrowers by debt size, FY2026 Q2.", + "", + header + " FSA |", + rule + "---:|", + ] + by_size = FSA["borrowers_by_debt_size_fy2026_q2"] + for label, _, _ in SCF_EDUCATION_DEBT_BALANCE_BANDS: + lines.append( + row(label, lambda t, k=label: t["borrowers"]["balance_bands"][k], _pct) + + f" {_pct(by_size[label] / sum(by_size.values()))} |" + ) + by_age = { + k: v + for k, v in FSA["borrowers_by_age_fy2026_q2"].items() + if k != "not_reported" + } + lines += [ + "", + "### Borrowers by FSA age band", + "", + "FSA: borrowers by age, FY2026 Q2, of those with an age reported.", + "", + header + " FSA |", + rule + "---:|", + ] + for label, _, _ in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: + lines.append( + row(label, lambda t, k=label: t["borrowers"]["age_bands"][k], _pct) + + f" {_pct(by_age[label] / sum(by_age.values()))} |" + ) + for section, title in ( + ( + "by_head_age_band", + "Households with federal education debt, by age of reference person", + ), + ("by_race", "Households with federal education debt, by race (`racecl4`)"), + ( + "by_income_quintile", + "Households with federal education debt, by income quintile", + ), + ): + lines += ["", f"### {title}", "", header, rule] + for group in observed[section]: + lines.append( + row( + group, + lambda t, s=section, g=group: t[s][g][ + "share_with_federal_education_debt" + ], + _pct, + ) + ) + lines += ["", "Median among borrowers:", "", header, rule] + for group in observed[section]: + lines.append( + row( + group, + lambda t, s=section, g=group: t[s][g]["median_among_borrowers"], + _dollars, + ) + ) + lines += [ + "", + "## Predictor ablation (five-fold hold-out on SCF families)", + "", + "Each variant refits both QRFs. Errors are drawn minus observed; TVD is " + "total variation distance between band distributions; MAE averages the " + "absolute error of the share with debt across the groups. `educ` is the " + "SCF reference person's education, which the CPS side cannot supply until " + "microcosm#297, so that row shows what the predictor would add. The last " + "column is the weighted share of held-out families observed with federal " + "debt but no Parent PLUS that are drawn some Parent PLUS anyway.", + "", + ] + metrics = [ + "share_with_debt_error_pp", + "mean_among_borrowers_error_pct", + "median_among_borrowers_error_pct", + "household_balance_band_tvd", + "head_age_band_share_mae_pp", + "race_share_mae_pp", + "income_quintile_share_mae_pp", + "borrower_balance_band_tvd", + "borrower_age_band_tvd", + "household_rank_correlation", + "parent_plus_drawn_without_observed_parent_plus", + ] + lines.append("| Variant | " + " | ".join(metrics) + " |") + lines.append("|---|" + "---:|" * len(metrics)) + for name, variant in receipt["holdout"]["variants"].items(): + values = variant["metrics"] + marker = " (build)" if name == PRODUCTION_VARIANT else "" + lines.append( + f"| {name}{marker} | " + + " | ".join(f"{values[m]:.3f}" for m in metrics) + + " |" + ) + if host is not None: + host_metrics = [ + m + for m in metrics + if m + not in { + "household_rank_correlation", + "parent_plus_drawn_without_observed_parent_plus", + } + ] + lines += [ + "", + "## Host fit, before and after", + "", + f"Imputed with build seed {receipt['host']['seed']} onto " + f"`{receipt['host']['release']}` ({receipt['host']['households']:,} " + f"households, sha256 `{receipt['host']['sha256'][:12]}...`) at its " + "calibrated household weights; the published weights were calibrated " + "without these columns. Build signal gate: " + f"{'passed' if receipt['host']['gate_passed'] else 'FAILED'}. Errors " + "are imputed minus SCF 2025.", + "", + "| Setting | Households with debt | Parent PLUS share | " + + " | ".join(host_metrics) + + " |", + "|---|---:|---:|" + "---:|" * len(host_metrics), + ] + for setting in settings.values(): + fit = setting["fit"] + lines.append( + f"| {setting['label']} | " + f"{_pct(fit['households']['share_with_federal_education_debt'])} | " + f"{_pct(fit['households']['parent_plus_share_of_total'])} | " + + " | ".join( + f"{setting['metrics_vs_scf'][m]:.3f}" for m in host_metrics + ) + + " |" + ) + lines.append( + "| SCF 2025 | " + f"{_pct(observed['households']['share_with_federal_education_debt'])} | " + f"{_pct(observed['households']['parent_plus_share_of_total'])} | " + + " | ".join("" for _ in host_metrics) + + " |" + ) + lines.append("") + return "\n".join(lines) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--scf-zip-dir", + type=Path, + help="Directory holding scfp2025s.zip and scf2025s.zip (else fetch).", + ) + parser.add_argument( + "--scf-cache-dir", + type=Path, + default=Path.home() / ".cache" / "microcosm" / "scf", + ) + parser.add_argument("--h5", type=Path, help="US Microcosm H5 to impute onto.") + parser.add_argument("--host-release", default=DEFAULT_HOST_RELEASE) + parser.add_argument( + "--run", + default="", + help="Label of the run that produced the receipt (the Modal run id).", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--folds", type=int, default=5) + parser.add_argument("--n-estimators", type=int, default=100) + parser.add_argument( + "--variants", + nargs="*", + default=list(VARIANTS), + choices=list(VARIANTS), + ) + parser.add_argument( + "--out-json", + type=Path, + default=Path("experiments/receipts/1073-scf-education-debt-fit.json"), + ) + parser.add_argument( + "--out-md", + type=Path, + default=Path("experiments/1073-scf-education-debt-receipts.md"), + ) + args = parser.parse_args(argv) + + if args.scf_zip_dir is not None: + summary_path = _member_from_local_zip( + SCF_2025_SUMMARY_EXTRACT, args.scf_zip_dir.expanduser(), args.scf_cache_dir + ) + full_path = _member_from_local_zip( + SCF_2025_FULL_EXTRACT, args.scf_zip_dir.expanduser(), args.scf_cache_dir + ) + else: + summary_path = fetch_scf_public_file( + SCF_2025_SUMMARY_EXTRACT, args.scf_cache_dir + ) + full_path = fetch_scf_public_file(SCF_2025_FULL_EXTRACT, args.scf_cache_dir) + + donor = load_scf_2025_education_debt_donor(summary_path, full_path) + full = _read_stata_columns( + full_path, FULL_FILE_RECEIPT_COLUMNS, label="2025 full file" + ) + full["wgt"] = full["x42001"] / 5.0 + observed = scf_education_debt_donor_fit_table(donor) + observed_before = scf_education_debt_donor_fit_table( + donor_under_reading(donor, full, "before") + ) + receipt: dict[str, object] = { + "schema_version": 1, + "issue": ISSUE, + "generated_at": datetime.now(UTC).strftime("%Y-%m-%d"), + "provenance": {"commit": _git_commit(), "run": args.run}, + "scf_files": { + public_file.member: { + "url": public_file.url, + "zip_sha256": public_file.zip_sha256, + "member_sha256": public_file.member_sha256, + } + for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT) + }, + "source_checks": { + **source_checks(donor, full), + "x8000_degree_check": x8000_degree_check(full), + "child_education_readings": child_education_readings(full), + "consolidated_share_of_federal_balances_dated_2014_or_later": ( + consolidated_share_of_new_loans(full) + ), + }, + "fsa": FSA, + "previous_receipt": PREVIOUS_RECEIPT, + "scf_2025_observed": observed, + "scf_2025_observed_before": observed_before, + "holdout": { + "folds": args.folds, + "seed": args.seed, + "n_estimators": args.n_estimators, + "production_variant": PRODUCTION_VARIANT, + "variants": {}, + }, + } + observed_total = donor[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(dtype=np.float64) + for name in args.variants: + predictors = VARIANTS[name] + drawn = holdout_draws( + donor, + predictors, + folds=args.folds, + seed=args.seed, + n_estimators=args.n_estimators, + ) + fit = scf_education_debt_donor_fit_table(donor, imputed=drawn) + metrics = fit_metrics(observed, fit) + metrics["household_rank_correlation"] = _rank_correlation( + observed_total, + drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(dtype=np.float64), + ) + metrics["parent_plus_drawn_without_observed_parent_plus"] = ( + parent_plus_draw_rate(donor, drawn) + ) + receipt["holdout"]["variants"][name] = { + "predictors": list(predictors), + "fit": fit, + "metrics": metrics, + } + print(name, json.dumps(metrics, indent=None, default=float)) + + if args.h5 is not None: + host_sha256 = _sha256(args.h5) + base = load_host_frame(args.h5) + receipt["host_settings"] = {} + for name, (label, definition, predictors) in HOST_SETTINGS.items(): + if name == BUILD_SETTING: + continue + imputed = impute_us_scf_education_debt( + base, + donor, + seed=args.seed, + n_estimators=args.n_estimators, + definition=definition, + predictors=predictors, + ) + tables = {entity: base.table(entity).copy() for entity in base.entities} + for column in imputed.columns: + tables["person"][column] = imputed[column].to_numpy() + setting_frame = Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + ) + setting_fit = us_scf_education_debt_fit_table(setting_frame, definition) + receipt["host_settings"][name] = { + "label": label, + "definition": asdict(definition), + "predictors": list(predictors), + "fit": setting_fit, + "metrics_vs_scf": fit_metrics(observed, setting_fit), + } + # The build runs through the stage entry point and its signal gate. + frame = with_us_scf_education_debt_inputs( + base, + seed=args.seed, + time_period=2024, + scf_education_debt_donor=donor, + n_estimators=args.n_estimators, + ) + gate = us_scf_education_debt_signal_gate(frame) + host_fit = us_scf_education_debt_fit_table(frame) + label, definition, predictors = HOST_SETTINGS[BUILD_SETTING] + receipt["host_settings"][BUILD_SETTING] = { + "label": label, + "definition": asdict(definition), + "predictors": list(predictors), + "fit": host_fit, + "metrics_vs_scf": fit_metrics(observed, host_fit), + } + receipt["host"] = { + "release": args.host_release, + "path": args.h5.name, + "sha256": host_sha256, + "is_the_published_default": host_sha256 == DEFAULT_HOST_SHA256, + "households": int(len(frame.table("household"))), + "people": int(len(frame.table("person"))), + "seed": args.seed, + "gate_passed": bool(gate.passed), + "gate_failures": list(gate.failures), + "household_income_columns_present": gate.details.get( + "household_income_columns_present" + ), + "fit": host_fit, + "metrics_vs_scf": fit_metrics(observed, host_fit), + } + print("host", json.dumps(receipt["host"]["metrics_vs_scf"], default=float)) + + args.out_json.parent.mkdir(parents=True, exist_ok=True) + args.out_json.write_text( + json.dumps(receipt, indent=2, sort_keys=False, default=float) + "\n", + encoding="utf-8", + newline="\n", + ) + args.out_md.write_text(markdown(receipt), encoding="utf-8", newline="\n") + print(f"wrote {args.out_json} and {args.out_md}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/diagnose_1073_host_fit.py b/experiments/diagnose_1073_host_fit.py new file mode 100644 index 0000000000..805c1d28ca --- /dev/null +++ b/experiments/diagnose_1073_host_fit.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python +"""Diagnose the SCF-vs-CPS-host gap of the education-debt stage (microcosm#1073). + +The first receipt imputed 14.4% of CPS host households with federal education +debt against 18.2% in SCF 2025 (17.3% in the SCF hold-out), short mostly at +reference-person ages 25-49 and income quintiles 2-5, and a Parent PLUS share +of 21.5% against 13.8%. A QRF reproduces its donor's conditional distribution, +so a host-only gap means the host's predictors are distributed or coded +differently from the donor's. This script measures that, weighted, before any +change: + +1. who the reference person is (SCF: the economically dominant individual or + couple, then the male or older partner; CPS candidates: the current + oldest-adult rule and the householder-anchored rule); +2. the predictors by reference-person age band: married-or-partnered (SCF + ``married`` counts partners), children of the reference person or partner + (SCF ``kids``), race, and income summed over candidate CPS units (household, + family, SPM unit, couple plus own children, couple only), against the SCF + primary economic unit's ``wageinc`` and ``income - kginc``. + + .venv/Scripts/python.exe experiments/diagnose_1073_host_fit.py \ + --h5 populace_us_2024.h5 +""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Mapping, Sequence +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS, + load_scf_2025_education_debt_donor, +) + +AGE_BANDS = (("18_to_24", 18, 24), ("25_to_34", 25, 34), ("35_to_49", 35, 49), + ("50_to_61", 50, 61), ("62_plus", 62, 200)) # fmt: skip +_HOST_COLUMNS = ( + "person_household_id", + "person_family_id", + "person_spm_unit_id", + "age", + "is_female", + "is_household_head", + "A_LINENO", + "A_SPOUSE", + "PECOHAB", + "A_MARITL", + "PEPAR1", + "PEPAR2", + "PRDTRACE", + "PRDTHSP", + "employment_income_before_lsr", +) + + +def wq(values: np.ndarray, weights: np.ndarray, q: float) -> float: + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, q * cumulative[-1], side="left")) + return float(values[order][min(index, len(values) - 1)]) + + +def summarize(values: np.ndarray, weights: np.ndarray) -> dict[str, float]: + return { + "p25": wq(values, weights, 0.25), + "p50": wq(values, weights, 0.50), + "p75": wq(values, weights, 0.75), + "p90": wq(values, weights, 0.90), + "mean": float(np.average(values, weights=weights)), + } + + +def share(flag: np.ndarray, weights: np.ndarray) -> float: + return float(weights[flag].sum() / weights.sum()) + + +def load_host(path: Path) -> tuple[pd.DataFrame, pd.Series]: + with pd.HDFStore(path, "r") as store: + available = set(store.get_storer("/person").data_columns) + columns = [c for c in _HOST_COLUMNS if c in available] + [ + c for c in _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS if c in available + ] + person = store.select("/person", columns=columns).reset_index(drop=True) + household = store.select( + "/household", columns=["household_id", "household_weight"] + ) + weights = pd.Series( + household["household_weight"].to_numpy(dtype=float), + index=household["household_id"].to_numpy(), + ) + return person, weights + + +def _num(series: pd.Series, fill: float = 0.0) -> np.ndarray: + return pd.to_numeric(series, errors="coerce").fillna(fill).to_numpy(dtype=float) + + +def host_units(person: pd.DataFrame) -> pd.DataFrame: + """One row per household: reference person under two rules, and units.""" + + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + age = _num(person["age"]) + female = person["is_female"].astype(bool).to_numpy() + lookup = {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + spouse_ptr = _num(person["A_SPOUSE"]).astype(int) + partner_ptr = _num(person["PECOHAB"]).astype(int) + married = np.isin(_num(person["A_MARITL"]).astype(int), (1, 2)) + + def partner_of(i: int) -> int: + for pointer in (spouse_ptr[i], partner_ptr[i]): + if pointer > 0: + j = lookup.get((hh[i], int(pointer))) + if j is not None and j != i: + return j + return -1 + + # Current rule (scf_auto_loans._scf_reference_person_mask): the only adult, + # the man of a two-adult married couple (older if same sex), else the + # oldest adult. Householder rule: the CPS householder (line 1) or, when the + # householder lives with a spouse or partner, the SCF's man-or-older of the + # two. + groups = pd.Series(np.arange(len(person))).groupby(hh, sort=False) + rows = [] + for household_id, positions in groups: + idx = positions.to_numpy() + adults = idx[age[idx] >= 18] + if len(adults) == 0: + current = idx[np.argmax(age[idx])] + elif len(adults) == 1: + current = adults[0] + elif len(adults) == 2 and married[adults].all(): + if female[adults[0]] == female[adults[1]]: + current = adults[np.argmax(age[adults])] + else: + current = adults[~female[adults]][0] + else: + current = adults[np.argmax(age[adults])] + householder = idx[np.argmin(line[idx])] + partner = partner_of(householder) + anchored = householder + if partner >= 0: + pair = np.array([householder, partner]) + if female[pair[0]] != female[pair[1]]: + anchored = pair[~female[pair]][0] + else: + anchored = pair[np.argmax(age[pair])] + rows.append((household_id, current, anchored, householder)) + return pd.DataFrame( + rows, columns=["household_id", "current", "anchored", "householder"] + ) + + +def partner_rows(person: pd.DataFrame, rows: np.ndarray) -> np.ndarray: + """Positional row of each given row's spouse (A_SPOUSE) or partner (PECOHAB).""" + + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + lookup = {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + pointers = ( + _num(person["A_SPOUSE"]).astype(int), + _num(person["PECOHAB"]).astype(int), + ) + out = np.full(len(rows), -1, dtype=np.int64) + for k, r in enumerate(rows): + for ptr in pointers: + if ptr[r] > 0: + j = lookup.get((hh[r], int(ptr[r]))) + if j is not None and j != r: + out[k] = j + break + return out + + +def unit_income( + person: pd.DataFrame, + reference: np.ndarray, + unit: str, + column_sets: Mapping[str, Sequence[str]], +) -> dict[str, np.ndarray]: + """Income sums over a candidate unit around each household's reference person.""" + + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + n = len(person) + member = np.zeros(n, dtype=bool) + owner = np.full(n, -1, dtype=np.int64) # household row index of the unit + ref_rows = reference + if unit == "household": + member[:] = True + index_of = pd.Series(np.arange(len(ref_rows)), index=hh[ref_rows]) + owner = index_of.reindex(hh).to_numpy() + else: + ref_set = {} + for k, r in enumerate(ref_rows): + ref_set[r] = k + partner = partner_rows(person, ref_rows) + if unit in ("family", "spm_unit"): + key = "person_family_id" if unit == "family" else "person_spm_unit_id" + ids = person[key].to_numpy() + unit_of_ref = pd.Series(np.arange(len(ref_rows)), index=ids[ref_rows]) + unit_of_ref = unit_of_ref[~unit_of_ref.index.duplicated()] + owner = unit_of_ref.reindex(ids).to_numpy(dtype=float) + owner = np.where(np.isnan(owner), -1, owner).astype(np.int64) + # a unit id can belong to another household's reference person only + # if ids repeat across households; guard on household + same = owner >= 0 + same[same] = hh[same] == hh[ref_rows[owner[same]]] + owner = np.where(same, owner, -1) + member = owner >= 0 + if unit == "family": + # a cohabiting partner is a separate CPS family; SCF counts + # partners in the PEU, so add the partner explicitly + for k, j in enumerate(partner): + if j >= 0 and owner[j] < 0: + owner[j] = k + member[j] = True + else: + for k, r in enumerate(ref_rows): + owner[r] = k + if partner[k] >= 0: + owner[partner[k]] = k + if unit == "couple_and_children": + parents = {(hh[r], line[r]): k for k, r in enumerate(ref_rows)} + for k, j in enumerate(partner): + if j >= 0: + parents[(hh[j], line[j])] = k + for pointer in ("PEPAR1", "PEPAR2"): + ptr = _num(person[pointer]).astype(int) + for i in range(n): + if owner[i] < 0 and ptr[i] > 0: + k = parents.get((hh[i], ptr[i])) + if k is not None: + owner[i] = k + member = owner >= 0 + out = {} + for name, columns in column_sets.items(): + values = np.zeros(n) + for column in columns: + if column in person.columns: + values += _num(person[column]) + if name == "employment_income": + values = np.maximum(values, 0.0) + sums = np.zeros(len(ref_rows)) + np.add.at(sums, owner[member], values[member]) + out[name] = sums + return out + + +def children_counts( + person: pd.DataFrame, reference: np.ndarray +) -> dict[str, np.ndarray]: + hh = person["person_household_id"].to_numpy() + line = _num(person["A_LINENO"], -1).astype(int) + {(h, n_): i for i, (h, n_) in enumerate(zip(hh, line, strict=True))} + age = _num(person["age"]) + ref_lines = {(hh[r], line[r]): k for k, r in enumerate(reference)} + partners = partner_rows(person, reference) + partner_lines = {(hh[j], line[j]): k for k, j in enumerate(partners) if j >= 0} + own = np.zeros(len(reference)) + either = np.zeros(len(reference)) + either_under_18 = np.zeros(len(reference)) + ptr1 = _num(person["PEPAR1"]).astype(int) + ptr2 = _num(person["PEPAR2"]).astype(int) + for i in range(len(person)): + keys = [(hh[i], p) for p in (ptr1[i], ptr2[i]) if p > 0] + if not keys: + continue + k_ref = next((ref_lines[x] for x in keys if x in ref_lines), None) + k_any = ( + k_ref + if k_ref is not None + else next((partner_lines[x] for x in keys if x in partner_lines), None) + ) + if k_ref is not None: + own[k_ref] += 1 + if k_any is not None: + either[k_any] += 1 + if age[i] < 18: + either_under_18[k_any] += 1 + return {"own_of_reference": own, "of_reference_or_partner": either, + "of_reference_or_partner_under_18": either_under_18} # fmt: skip + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--h5", type=Path, required=True) + parser.add_argument( + "--scf-cache-dir", type=Path, default=Path.home() / ".cache/microcosm/scf" + ) + parser.add_argument( + "--out", + type=Path, + default=Path("experiments/receipts/1073-host-fit-diagnosis.json"), + ) + args = parser.parse_args(argv) + + donor = load_scf_2025_education_debt_donor( + args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" + ) + scf_w = donor["scf_weight"].to_numpy(float) + scf_age = donor["age"].to_numpy(float) + + person, hh_weights = load_host(args.h5) + units = host_units(person) + weights = hh_weights.reindex(units["household_id"]).to_numpy(float) + age = _num(person["age"]) + report: dict[str, object] = {} + + # 1. Who is the reference person, and how old. + report["reference_person_age_bands"] = { + "scf": { + b: share((scf_age >= lo) & (scf_age <= hi), scf_w) + for b, lo, hi in AGE_BANDS + }, + **{ + f"cps_{rule}": { + b: share((age[units[rule]] >= lo) & (age[units[rule]] <= hi), weights) + for b, lo, hi in AGE_BANDS + } + for rule in ("current", "anchored") + }, + } + report["rules_disagree_share"] = share( + units["current"].to_numpy() != units["anchored"].to_numpy(), weights + ) + + column_sets = { + "employment_income": ("employment_income_before_lsr",), + "household_income": _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS, + } + for rule in ("current", "anchored"): + reference = units[rule].to_numpy() + ref_age = age[reference] + has_partner = partner_rows(person, reference) >= 0 + married_code = np.isin(_num(person["A_MARITL"]).astype(int)[reference], (1, 2)) + kids = children_counts(person, reference) + incomes = { + unit: unit_income(person, reference, unit, column_sets) + for unit in ( + "household", + "spm_unit", + "family", + "couple_and_children", + "couple", + ) + } + by_band = {} + for band, lo, hi in AGE_BANDS: + host = (ref_age >= lo) & (ref_age <= hi) + scf = (scf_age >= lo) & (scf_age <= hi) + entry = { + "scf_married_or_partnered": share( + donor["is_married"].to_numpy()[scf] == 1, scf_w[scf] + ), + "cps_A_MARITL_1_2": share(married_code[host], weights[host]), + "cps_spouse_or_partner_present": share( + has_partner[host], weights[host] + ), + "scf_kids_positive": share( + donor["own_children_in_household"].to_numpy()[scf] > 0, + scf_w[scf], + ), + **{ + f"cps_kids_{k}_positive": share(v[host] > 0, weights[host]) + for k, v in kids.items() + }, + "scf_employment_income": summarize( + donor["employment_income"].to_numpy()[scf], scf_w[scf] + ), + "scf_household_income": summarize( + donor["household_income"].to_numpy()[scf], scf_w[scf] + ), + } + for unit, sums in incomes.items(): + for name, values in sums.items(): + entry[f"cps_{name}_{unit}"] = summarize(values[host], weights[host]) + by_band[band] = entry + report[f"predictors_by_reference_age_{rule}_rule"] = by_band + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + + # Console digest + print("ref age bands", json.dumps(report["reference_person_age_bands"], indent=0)) + print("rules disagree", report["rules_disagree_share"]) + for rule in ("current", "anchored"): + for band in ("25_to_34", "35_to_49", "50_to_61"): + e = report[f"predictors_by_reference_age_{rule}_rule"][band] + print(f"\n[{rule}] {band}") + for key in ( + "scf_married_or_partnered", + "cps_A_MARITL_1_2", + "cps_spouse_or_partner_present", + "scf_kids_positive", + "cps_kids_own_of_reference_positive", + "cps_kids_of_reference_or_partner_positive", + ): + print(f" {key}: {e[key]:.3f}") + for name in ("employment_income", "household_income"): + s = e[f"scf_{name}"] + print( + f" scf {name}: p25 {s['p25']:,.0f} p50 {s['p50']:,.0f} p75 {s['p75']:,.0f} p90 {s['p90']:,.0f}" + ) + for unit in ( + "household", + "spm_unit", + "family", + "couple_and_children", + "couple", + ): + s = e[f"cps_{name}_{unit}"] + print( + f" cps {name} {unit}: p25 {s['p25']:,.0f} p50 {s['p50']:,.0f} p75 {s['p75']:,.0f} p90 {s['p90']:,.0f}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/diagnose_1073_predictor_shift.py b/experiments/diagnose_1073_predictor_shift.py new file mode 100644 index 0000000000..66933267b8 --- /dev/null +++ b/experiments/diagnose_1073_predictor_shift.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +"""Which predictor's SCF-vs-CPS shift drives the host borrower gap (microcosm#1073). + +Fits the stage's household-total QRF once on the SCF 2025 donor, draws the CPS +host under the stage's recipient definition, then redraws it with one +predictor at a time replaced by its SCF counterpart at the same weighted rank +within the same five-year reference-person age bin (a quantile mapping), and +with all of them replaced. A predictor whose mapping closes the gap is one the +two surveys distribute differently at equal rank. Also writes each predictor's +weighted distribution by age band. Output: +``experiments/receipts/1073-predictor-shift-diagnosis.json``. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +from collections.abc import Sequence +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + RecipientDefinition, + _recipients, + _stage_seed, + load_scf_2025_education_debt_donor, +) + +_HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "receipt_1073", _HERE / "build_1073_scf_education_debt_receipt.py" +) +receipt = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(receipt) + +#: The construction this diagnosis measured: the SCF-matching codings with +#: whole-household income, before the predictor and income-unit remedies it led +#: to (receipt run 20261020T150445Z-47949111f). Pinned so a rerun keeps its +#: meaning now that the stage default has moved on. +DIAGNOSED_DEFINITION = RecipientDefinition(income_unit="household") +BANDS = (("18_24", 18, 24), ("25_34", 25, 34), ("35_49", 35, 49), + ("50_61", 50, 61), ("62_plus", 62, 200)) # fmt: skip +CONTINUOUS = ( + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income", + "own_children_in_household", +) + + +def wq(values: np.ndarray, weights: np.ndarray, q: float) -> float: + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, q * cumulative[-1], side="left")) + return float(values[order][min(index, len(values) - 1)]) + + +def describe(values: np.ndarray, weights: np.ndarray, *, binary: bool) -> dict: + if binary: + return {"share": float(weights[values > 0].sum() / weights.sum())} + return { + "share_positive": float(weights[values > 0].sum() / weights.sum()), + **{ + f"p{int(q * 100)}": wq(values, weights, q) + for q in (0.25, 0.5, 0.75, 0.9, 0.99) + }, + } + + +def weighted_rank(values: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Mid-rank of each value in its weighted distribution, ties sharing a rank.""" + + frame = pd.DataFrame({"v": values, "w": weights}) + by_value = frame.groupby("v")["w"].sum().sort_index() + upper = by_value.cumsum() / by_value.sum() + mid = upper - by_value / by_value.sum() / 2.0 + return mid.reindex(values).to_numpy() + + +def quantile_map( + host: np.ndarray, + host_w: np.ndarray, + host_age: np.ndarray, + scf: np.ndarray, + scf_w: np.ndarray, + scf_age: np.ndarray, +) -> np.ndarray: + """Replace each host value by the SCF value at its rank in its age bin.""" + + out = host.copy() + bins = np.arange(15, 100, 5) + host_bin = np.digitize(host_age, bins) + scf_bin = np.digitize(scf_age, bins) + for b in np.unique(host_bin): + h = host_bin == b + s = scf_bin == b + if s.sum() < 20: + s = np.abs(scf_bin - b) <= 1 + ranks = weighted_rank(host[h], host_w[h]) + order = np.argsort(scf[s], kind="mergesort") + cumulative = np.cumsum(scf_w[s][order]) / scf_w[s].sum() + index = np.searchsorted(cumulative, ranks, side="left") + out[h] = scf[s][order][np.minimum(index, s.sum() - 1)] + return out + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--h5", type=Path, required=True) + parser.add_argument( + "--scf-cache-dir", type=Path, default=Path.home() / ".cache/microcosm/scf" + ) + parser.add_argument( + "--out", + type=Path, + default=Path("experiments/receipts/1073-predictor-shift-diagnosis.json"), + ) + args = parser.parse_args(argv) + + from microcosm.fit import QRF + + donor = load_scf_2025_education_debt_donor( + args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" + ) + frame = receipt.load_host_frame(args.h5) + host = _recipients(frame, DIAGNOSED_DEFINITION).features + host_w = np.asarray(frame.weights_for("household").values, dtype=float) + host_age = host["age"].to_numpy(float) + scf_w = donor["scf_weight"].to_numpy(float) + scf_age = donor["age"].to_numpy(float) + + report: dict[str, object] = {"distributions": {}, "counterfactual_shares": {}} + for band, lo, hi in BANDS: + h = (host_age >= lo) & (host_age <= hi) + s = (scf_age >= lo) & (scf_age <= hi) + entry = {} + for column in SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS: + if column == "age": + continue + binary = column in ("is_female", "is_married") + if column == "cps_race": + entry[column] = { + "scf": { + str(int(c)): float(scf_w[s][donor[column].to_numpy()[s] == c].sum() / scf_w[s].sum()) + for c in (1, 2, 3, 4, 7) + }, + "cps": { + str(int(c)): float(host_w[h][host[column].to_numpy()[h] == c].sum() / host_w[h].sum()) + for c in (1, 2, 3, 4, 7) + }, + } # fmt: skip + continue + entry[column] = { + "scf": describe(donor[column].to_numpy(float)[s], scf_w[s], binary=binary), + "cps": describe(host[column].to_numpy(float)[h], host_w[h], binary=binary), + } # fmt: skip + report["distributions"][band] = entry + + predictors = list(SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS) + fit = donor.loc[:, [*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET, "scf_weight"]] + model = QRF(n_estimators=100, seed=_stage_seed(0, 1)).fit( + fit, + predictors=predictors, + targets=[SCF_EDUCATION_DEBT_TOTAL_TARGET], + weights="scf_weight", + ) + + def shares(features: pd.DataFrame) -> dict[str, float]: + drawn = model.predict(features.loc[:, predictors])[ + SCF_EDUCATION_DEBT_TOTAL_TARGET + ].to_numpy() + positive = drawn > 0 + out = {"all": float(host_w[positive].sum() / host_w.sum())} + for band, lo, hi in BANDS: + b = (host_age >= lo) & (host_age <= hi) + out[band] = float(host_w[b & positive].sum() / host_w[b].sum()) + return out + + scf_positive = donor[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy() > 0 + report["scf_observed_shares"] = { + "all": float(scf_w[scf_positive].sum() / scf_w.sum()), + **{ + band: float( + scf_w[scf_positive & (scf_age >= lo) & (scf_age <= hi)].sum() + / scf_w[(scf_age >= lo) & (scf_age <= hi)].sum() + ) + for band, lo, hi in BANDS + }, + } + report["counterfactual_shares"]["host_as_is"] = shares(host) + mapped_all = host.copy() + for column in CONTINUOUS: + mapped = host.copy() + mapped[column] = quantile_map( + host[column].to_numpy(float), + host_w, + host_age, + donor[column].to_numpy(float), + scf_w, + scf_age, + ) + mapped_all[column] = mapped[column] + report["counterfactual_shares"][f"map_{column}"] = shares(mapped) + print( + column, + json.dumps(report["counterfactual_shares"][f"map_{column}"]), + flush=True, + ) + report["counterfactual_shares"]["map_all_continuous"] = shares(mapped_all) + print("host", json.dumps(report["counterfactual_shares"]["host_as_is"])) + print("all", json.dumps(report["counterfactual_shares"]["map_all_continuous"])) + print("scf", json.dumps(report["scf_observed_shares"])) + for band in ("25_34", "35_49"): + print(band, json.dumps(report["distributions"][band], default=float)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps(report, indent=2, default=float) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/explore_1073_recipient_definitions.py b/experiments/explore_1073_recipient_definitions.py new file mode 100644 index 0000000000..9107129ef5 --- /dev/null +++ b/experiments/explore_1073_recipient_definitions.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +"""Host fit under each CPS construction of the SCF predictors (microcosm#1073). + +Each variant changes one field of ``RecipientDefinition`` from the one before, +for a stated reason (``VARIANTS``), and imputes the published default with the +same seed and donor through the production code path. Writes +``experiments/receipts/1073-recipient-definition-variants.json``. + + .venv/Scripts/python.exe experiments/explore_1073_recipient_definitions.py \ + --h5 populace_us_2024.h5 +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +from collections.abc import Sequence +from dataclasses import asdict, replace +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + RecipientDefinition, + _recipients, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, +) +from microcosm.frame import Frame + +_HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "receipt_1073", _HERE / "build_1073_scf_education_debt_receipt.py" +) +receipt = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(receipt) + +FIRST = SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION +_V1 = replace(FIRST, partnered="spouse_or_partner") +_V2 = replace(_V1, children="of_reference_or_partner") +_V3 = replace(_V2, reference_person="householder_couple") +VARIANTS: tuple[tuple[str, RecipientDefinition, str], ...] = ( + ("v0_first_receipt", FIRST, "The first receipt's construction."), + ( + "v1_partner_counts_as_married", + _V1, + "SCF MARRIED is married or living with a partner (X8023 in 1-2); " + "A_MARITL 1-2 drops cohabiting partners.", + ), + ( + "v2_children_of_reference_or_partner", + _V2, + "SCF KIDS counts children of the reference person or spouse/partner, " + "any age, anywhere in the household listing.", + ), + ( + "v3_householder_couple_reference_person", + _V3, + "The SCF reference person comes from the economically dominant " + "individual or couple (the man, or the older partner); the CPS " + "householder anchors that unit, where the oldest adult can be a " + "co-resident parent or roommate.", + ), + ( + "v4a_spm_unit_income", + replace(_V3, income_unit="spm_unit"), + "SCF income covers the primary economic unit; the SPM unit is the CPS " + "resource-sharing unit.", + ), + ( + "v4b_couple_and_children_income", + replace(_V3, income_unit="reference_couple_and_children"), + "The PEU narrowed to the couple and their children.", + ), +) +_ALL = tuple(SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS) +_NO_INTDIV = tuple(p for p in _ALL if p != "interest_dividend_income") +_NO_INTDIV_SSRET = tuple(p for p in _NO_INTDIV if p != "social_security_pension_income") +#: Predictor-set remedies on top of the couple income unit. Each row also gets +#: an SCF hold-out so the dropped predictor's own contribution is measured. +PREDICTOR_VARIANTS: tuple[ + tuple[str, RecipientDefinition, tuple[str, ...], str], ... +] = ( + ( + "r1_couple_income_drop_interest_dividends", + replace(_V3, income_unit="reference_couple_and_children"), + _NO_INTDIV, + "CPS interest and dividends are PUF-imputed at tax-unit level; at " + "reference ages 25-34, 42% of host households have some against 16% " + "of SCF families, so equal values do not mean equal positions.", + ), + ( + "r2_couple_income_drop_interest_dividends_and_ss_pension", + replace(_V3, income_unit="reference_couple_and_children"), + _NO_INTDIV_SSRET, + "As r1, and Social Security or pension income: 18% of host households " + "at 25-34 have some against 6% of SCF families.", + ), + ( + "r3_household_income_drop_interest_dividends_and_ss_pension", + _V3, + _NO_INTDIV_SSRET, + "As r2 with the household income unit, to separate the two changes.", + ), +) + + +def parent_plus_by_age( + head_age: np.ndarray, parent_plus: np.ndarray, total: np.ndarray, weight: np.ndarray +) -> dict[str, float]: + out = {} + for label, low, high in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: + inside = (head_age >= low) & (head_age <= high) + denominator = float(np.dot(total[inside], weight[inside])) + out[label] = ( + float(np.dot(parent_plus[inside], weight[inside])) / denominator + if denominator > 0 + else float("nan") + ) + return out + + +def host_parent_plus_by_age(frame: Frame, definition: RecipientDefinition) -> dict: + person = frame.table("person") + recipients = _recipients(frame, definition) + household_ids = person["person_household_id"].to_numpy() + order = frame.table("household")["household_id"].to_numpy() + own = person["federal_student_loan_balance"].to_numpy(dtype=float) + plus = person["federal_parent_plus_loan_balance"].to_numpy(dtype=float) + total = ( + pd.Series(own + plus, index=household_ids) + .groupby(level=0) + .sum() + .reindex(order) + .to_numpy() + ) + plus_hh = ( + pd.Series(plus, index=household_ids).groupby(level=0).sum().reindex(order) + ).to_numpy() + return parent_plus_by_age( + recipients.features["age"].to_numpy(dtype=float), + plus_hh, + total, + np.asarray(frame.weights_for("household").values, dtype=float), + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--h5", type=Path, required=True) + parser.add_argument( + "--scf-cache-dir", type=Path, default=Path.home() / ".cache/microcosm/scf" + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--n-estimators", type=int, default=100) + parser.add_argument( + "--out", + type=Path, + default=Path("experiments/receipts/1073-recipient-definition-variants.json"), + ) + args = parser.parse_args(argv) + + donor = load_scf_2025_education_debt_donor( + args.scf_cache_dir / "rscfp2025.dta", args.scf_cache_dir / "p25i6.dta" + ) + observed = scf_education_debt_donor_fit_table(donor) + scf_parent_plus = parent_plus_by_age( + donor["age"].to_numpy(float), + donor["federal_parent_plus_loan_balance"].to_numpy(float), + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy(float), + donor["scf_weight"].to_numpy(float), + ) + base = receipt.load_host_frame(args.h5) + results = { + "scf_2025": { + "share": observed["households"]["share_with_federal_education_debt"], + "parent_plus_share": observed["households"]["parent_plus_share_of_total"], + "parent_plus_by_reference_age": scf_parent_plus, + "by_head_age_band": { + k: v["share_with_federal_education_debt"] + for k, v in observed["by_head_age_band"].items() + }, + "by_income_quintile": { + k: v["share_with_federal_education_debt"] + for k, v in observed["by_income_quintile"].items() + }, + }, + "variants": {}, + } + holdout: dict[tuple[str, ...], dict] = {} + runs = [(n, d, _ALL, r) for n, d, r in VARIANTS] + list(PREDICTOR_VARIANTS) + for name, definition, predictors, reason in runs: + if predictors not in holdout: + drawn = receipt.holdout_draws( + donor, + predictors, + folds=5, + seed=args.seed, + n_estimators=args.n_estimators, + ) + holdout_fit = scf_education_debt_donor_fit_table(donor, imputed=drawn) + holdout[predictors] = { + "predictors": list(predictors), + "share": holdout_fit["households"]["share_with_federal_education_debt"], + "parent_plus_share": holdout_fit["households"][ + "parent_plus_share_of_total" + ], + "metrics_vs_scf": receipt.fit_metrics(observed, holdout_fit), + } + print("holdout", json.dumps(holdout[predictors], default=float), flush=True) + imputed = impute_us_scf_education_debt( + base, + donor, + seed=args.seed, + n_estimators=args.n_estimators, + definition=definition, + predictors=predictors, + ) + tables = {entity: base.table(entity).copy() for entity in base.entities} + for column in imputed.columns: + tables["person"][column] = imputed[column].to_numpy() + frame = Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + ) + fit = us_scf_education_debt_fit_table(frame, definition) + metrics = receipt.fit_metrics(observed, fit) + entry = { + "definition": asdict(definition), + "predictors": list(predictors), + "scf_holdout": holdout[predictors], + "reason": reason, + "share": fit["households"]["share_with_federal_education_debt"], + "parent_plus_share": fit["households"]["parent_plus_share_of_total"], + "parent_plus_by_reference_age": host_parent_plus_by_age(frame, definition), + "by_head_age_band": { + k: v["share_with_federal_education_debt"] + for k, v in fit["by_head_age_band"].items() + }, + "by_income_quintile": { + k: v["share_with_federal_education_debt"] + for k, v in fit["by_income_quintile"].items() + }, + "metrics_vs_scf": metrics, + "fit": fit, + } + results["variants"][name] = entry + print( + name, + json.dumps( + { + "share": round(entry["share"], 4), + "pp": round(entry["parent_plus_share"], 4), + "age": { + k: round(v, 3) for k, v in entry["by_head_age_band"].items() + }, + "inc": { + k: round(v, 3) for k, v in entry["by_income_quintile"].items() + }, + "pp_age": { + k: round(v, 3) + for k, v in entry["parent_plus_by_reference_age"].items() + }, + "age_mae": round(metrics["head_age_band_share_mae_pp"], 2), + "inc_mae": round(metrics["income_quintile_share_mae_pp"], 2), + } + ), + flush=True, + ) + print("scf", json.dumps(results["scf_2025"], default=float)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps(results, indent=2, default=float) + "\n", + encoding="utf-8", + newline="\n", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/modal_1073.py b/experiments/modal_1073.py new file mode 100644 index 0000000000..8c636912f6 --- /dev/null +++ b/experiments/modal_1073.py @@ -0,0 +1,520 @@ +"""Run the microcosm#1073 education-debt experiments on Modal instead of local RAM. + +Each run clones a pushed commit of PolicyEngine/microcosm, syncs that tree's +uv.lock, fetches the inputs and verifies each sha256, runs the requested steps +in order, and saves every receipt the steps write plus a run log to the +`microcosm-1073-cache` volume under results//. `collect` copies a +run's files back into this checkout. Inputs: + +- the published host release `populace-us-2024-spm-20260915` from Hugging Face + (policyengine/populace-us at a pinned revision); +- the Federal Reserve's SCF 2025 public zips (summary extract and full file). + + export PYTHONIOENCODING=utf-8 PYTHONUTF8=1 + modal run experiments/modal_1073.py::preflight --commit + modal run --detach experiments/modal_1073.py::launch --commit --steps tests,shift,explore + modal run experiments/modal_1073.py::collect --run-id + +Steps (see STEPS; each runs one or more commands, stopping at the first that +fails): + +- ``generate``: regenerate the US spec bundle mirrors (``spec/*.yaml``, + ``country_package.json``, ``engine_abi.lock.json``) with + ``tools/generate_us_bundle_from_constants.py`` after a hand edit of + ``source_stages.json``; +- ``tests``: the stage's unit tests; +- ``spec``: the bundle generator and the spec-engine coverage attestation in + ``--check`` mode, then the engine-free and engine-contract tests that read + the spec bundle, the stored-input register and the SCF stages, plus the + release tool's education-debt tests; +- ``hostfit``, ``shift``, ``explore``: the #1073 diagnostics; +- ``receipt``: the fit receipt, labelled with the run id. + +After the steps the run saves every tracked file they changed +(``git status --porcelain``) and every untracked file matching OUTPUT_GLOBS, +so ``collect`` brings regenerated mirrors back as well as receipts. This is an +experiment runner, not the registered stage runner in tools/modal_us_stage.py; +it writes nothing to the Hub and publishes nothing. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import time +import urllib.request +import zipfile +from datetime import UTC, datetime +from pathlib import Path + +import modal + +REPO_URL = "https://github.com/PolicyEngine/microcosm.git" +HOST = { + "repo_id": "policyengine/populace-us", + "filename": "populace_us_2024.h5", + "revision": "8ab57ffc2ca41d8af631ff62ebbce95e966299f3", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", +} +# zip name -> (url, zip sha256, member, member sha256) +SCF_ZIPS = { + "scfp2025s.zip": ( + "https://www.federalreserve.gov/econres/files/scfp2025s.zip", + "32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + "rscfp2025.dta", + "454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd", + ), + "scf2025s.zip": ( + "https://www.federalreserve.gov/econres/files/scf2025s.zip", + "c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + "p25i6.dta", + "38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad", + ), +} +_TESTS = "packages/microcosm-build/tests" +#: step -> commands. A part "glob:" expands to the sorted matches. +STEPS: dict[str, list[list[str]]] = { + "generate": [ + [ + "{python}", + "tools/generate_us_bundle_from_constants.py", + "--engine-lock-only", + ], + ["{python}", "tools/generate_us_bundle_from_constants.py"], + ], + "tests": [ + [ + "{python}", + "-m", + "pytest", + "-q", + f"{_TESTS}/engine_free/us/test_us_scf_education_debt.py", + ] + ], + "spec": [ + ["{python}", "tools/generate_us_bundle_from_constants.py", "--check"], + ["{python}", "tools/spec_engine_coverage.py", "--check"], + [ + "{python}", + "-m", + "pytest", + "-q", + f"glob:{_TESTS}/engine_free/shared/test_spec_engine_*.py", + f"{_TESTS}/engine_free/us/test_us_bundle_core_contracts.py", + f"{_TESTS}/engine_free/us/test_us_spec_bundle.py", + f"{_TESTS}/engine_free/us/test_us_stored_input_register.py", + f"{_TESTS}/engine_free/us/test_us_scf_auto_loans.py", + f"{_TESTS}/engine_free/us/test_us_scf_wealth.py", + f"{_TESTS}/engine_free/us/test_us_plan.py", + f"{_TESTS}/engine_free/us/test_us_spine_blindness.py", + ], + [ + "{python}", + "-m", + "pytest", + "-q", + f"{_TESTS}/engine_free/us/test_us_fiscal_refresh_builder.py", + "-k", + "scf_education_debt", + ], + [ + "{python}", + "-m", + "pytest", + "-q", + f"glob:{_TESTS}/engine_contract/us/test_spec_engine_*.py", + f"{_TESTS}/engine_contract/us/test_us_bundle_core_contracts.py", + f"{_TESTS}/engine_contract/us/test_us_spec_bundle.py", + f"{_TESTS}/engine_contract/us/test_us_stored_input_register.py", + f"{_TESTS}/engine_contract/us/test_us_scf_wealth.py", + f"{_TESTS}/engine_contract/us/test_imputation_lineage_spec.py", + f"{_TESTS}/engine_contract/us/test_release_input_coverage.py", + ], + ], + "hostfit": [ + [ + "{python}", + "experiments/diagnose_1073_host_fit.py", + "--h5", + "{h5}", + "--scf-cache-dir", + "{scf}", + ] + ], + "shift": [ + [ + "{python}", + "experiments/diagnose_1073_predictor_shift.py", + "--h5", + "{h5}", + "--scf-cache-dir", + "{scf}", + ] + ], + "explore": [ + [ + "{python}", + "experiments/explore_1073_recipient_definitions.py", + "--h5", + "{h5}", + "--scf-cache-dir", + "{scf}", + ] + ], + "receipt": [ + [ + "{python}", + "experiments/build_1073_scf_education_debt_receipt.py", + "--h5", + "{h5}", + "--scf-zip-dir", + "{zips}", + "--scf-cache-dir", + "{scf}", + "--run", + "{run_id}", + ] + ], +} +OUTPUT_GLOBS = ("experiments/receipts/1073-*.json", "experiments/1073-*.md") +CPU = 8.0 +MEMORY_MB = 32768 + +image = ( + modal.Image.debian_slim(python_version="3.14") + .apt_install("git") + .pip_install("uv==0.11.7", "huggingface_hub") +) +app = modal.App("microcosm-1073-experiments") +cache = modal.Volume.from_name("microcosm-1073-cache", create_if_missing=True) +CACHE = Path("/cache") +WORK = Path("/work/microcosm") +VENV = Path("/opt/venv") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def _verified(path: Path, sha256: str) -> bool: + return path.exists() and _sha256(path) == sha256 + + +def _clean_env(**overrides: str) -> dict[str, str]: + """The container environment without the Modal runtime's Python path. + + Modal puts its own client and dependencies (under /pkg) on PYTHONPATH. A + subprocess that inherits it sees those packages beside the venv's, and the + build's worker-identity check refuses the duplicates (e.g. idna). + """ + env = { + key: value + for key, value in os.environ.items() + if key not in {"PYTHONPATH", "PYTHONHOME", "PYTHONSTARTUP"} + } + env.update(overrides) + return env + + +def _checkout(commit: str) -> None: + WORK.mkdir(parents=True, exist_ok=True) + run = lambda *args: subprocess.run(args, cwd=WORK, check=True) # noqa: E731 + run("git", "init", "-q") + run("git", "remote", "add", "origin", REPO_URL) + run("git", "fetch", "-q", "--depth", "1", "origin", commit) + run("git", "checkout", "-q", "--detach", "FETCH_HEAD") + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=WORK, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if head != commit: + raise RuntimeError(f"checked out {head}, expected {commit}") + env = _clean_env( + **{ + "UV_PROJECT_ENVIRONMENT": str(VENV), + # A Modal volume cannot host uv's cache: uv persists temporary files + # by atomic rename, which the volume refuses. + "UV_CACHE_DIR": "/root/.cache/uv", + "UV_LINK_MODE": "copy", + } + ) + subprocess.run( + ["uv", "sync", "--all-packages", "--extra", "us", "--frozen"], + cwd=WORK, + env=env, + check=True, + ) + + +def _inputs() -> dict[str, Path]: + from huggingface_hub import hf_hub_download + + host_dir = CACHE / "host" / HOST["revision"] + host = host_dir / HOST["filename"] + if not _verified(host, HOST["sha256"]): + hf_hub_download( + repo_id=HOST["repo_id"], + filename=HOST["filename"], + revision=HOST["revision"], + repo_type="dataset", + local_dir=host_dir, + ) + if not _verified(host, HOST["sha256"]): + raise RuntimeError("host H5 sha256 mismatch") + zips = CACHE / "scf" / "zips" + scf = CACHE / "scf" / "dta" + zips.mkdir(parents=True, exist_ok=True) + scf.mkdir(parents=True, exist_ok=True) + for name, (url, zip_sha, member, member_sha) in SCF_ZIPS.items(): + archive = zips / name + if not _verified(archive, zip_sha): + request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(request, timeout=600) as response: + archive.write_bytes(response.read()) + if not _verified(archive, zip_sha): + raise RuntimeError(f"{name} sha256 mismatch") + target = scf / member + if not _verified(target, member_sha): + with zipfile.ZipFile(archive) as bundle: + target.write_bytes(bundle.read(member)) + if not _verified(target, member_sha): + raise RuntimeError(f"{member} sha256 mismatch") + cache.commit() + return {"h5": host, "zips": zips, "scf": scf} + + +def _expand(command: list[str], fields: dict[str, str]) -> list[str]: + parts: list[str] = [] + for part in command: + if part.startswith("glob:"): + matches = sorted(WORK.glob(part.removeprefix("glob:"))) + if not matches: + raise RuntimeError(f"{part} matched nothing") + parts.extend(str(path.relative_to(WORK)) for path in matches) + else: + parts.append(part.format(**fields)) + return parts + + +def _save_changed_files(target_root: Path) -> list[str]: + """Copy tracked files the steps changed, and new outputs, under target_root.""" + + status = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=WORK, + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + saved = [] + for line in status: + code, rel = line[:2], line[3:] + path = WORK / rel + untracked = code == "??" + if untracked and not any(path.match(p) for p in OUTPUT_GLOBS): + continue + if not path.is_file(): + continue + target = target_root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(path.read_bytes()) + saved.append(f"{code} {rel}") + return saved + + +@app.function( + image=image, cpu=2.0, memory=8192, timeout=3600, volumes={"/cache": cache} +) +def preflight_remote(commit: str) -> dict: + """Clone, sync and stage the inputs without running a step.""" + started = time.time() + _checkout(commit) + paths = _inputs() + probe = subprocess.run( + [ + str(VENV / "bin" / "python"), + "-c", + "import microcosm.build.us_runtime.scf_education_debt as m; print(m.__file__)", + ], + cwd=WORK, + capture_output=True, + text=True, + ) + return { + "commit": commit, + "inputs": {key: str(path) for key, path in paths.items()}, + "import_ok": probe.returncode == 0, + "import_output": (probe.stdout + probe.stderr)[-2000:], + "seconds": round(time.time() - started, 1), + } + + +@app.function( + image=image, cpu=CPU, memory=MEMORY_MB, timeout=4 * 3600, volumes={"/cache": cache} +) +def run_steps( + commit: str, steps: list[str], run_id: str, script_text: str = "" +) -> dict: + import resource # Unix-only; the local entrypoint runs on Windows too + + started = datetime.now(UTC) + _checkout(commit) + paths = _inputs() + fields = {key: str(path) for key, path in paths.items()} | { + "python": str(VENV / "bin" / "python"), + "run_id": run_id, + } + threads = str(int(CPU)) + env = _clean_env( + PYTHONUNBUFFERED="1", + OMP_NUM_THREADS=threads, + MKL_NUM_THREADS=threads, + OPENBLAS_NUM_THREADS=threads, + ) + out = CACHE / "results" / run_id + out.mkdir(parents=True, exist_ok=True) + log = { + "run_id": run_id, + "commit": commit, + "steps": [], + "cpu": CPU, + "memory_mb": MEMORY_MB, + "inputs": {"host": HOST, "scf": {k: v[1] for k, v in SCF_ZIPS.items()}}, + "started_at": started.isoformat(), + } + failed = False + for step in steps: + if step == "script": + # An uncommitted local probe passed by --script; it runs in the + # clone like any step, and the run log records its text. + (WORK / "_modal_probe.py").write_text(script_text, encoding="utf-8") + log["script_sha256"] = hashlib.sha256(script_text.encode()).hexdigest() + commands = [[fields["python"], "_modal_probe.py"]] + else: + commands = [_expand(command, fields) for command in STEPS[step]] + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + t0 = time.time() + returncodes = [] + with (out / f"{step}.log").open("w", encoding="utf-8") as handle: + for command in commands: + handle.write("$ " + " ".join(command) + "\n") + handle.flush() + result = subprocess.run( + command, + cwd=WORK, + env=env, + stdout=handle, + stderr=subprocess.STDOUT, + text=True, + ) + returncodes.append(result.returncode) + if result.returncode != 0: + break + peak_kb = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + log["steps"].append( + { + "step": step, + "commands": commands, + "returncodes": returncodes, + "returncode": next((code for code in returncodes if code), 0), + "seconds": round(time.time() - t0, 1), + "peak_child_rss_gb": round(max(peak_kb, before) / 1024 / 1024, 2), + } + ) + cache.commit() + if any(returncodes): + failed = True + break + log["failed"] = failed + log["changed_files"] = _save_changed_files(out / "files") + log["finished_at"] = datetime.now(UTC).isoformat() + (out / "run.json").write_text(json.dumps(log, indent=2) + "\n", encoding="utf-8") + cache.commit() + return log + + +@app.function(image=image, volumes={"/cache": cache}) +def fetch_results(run_id: str) -> dict[str, bytes]: + root = CACHE / "results" / run_id + return { + str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file() + } + + +def _check_commit(commit: str) -> None: + if len(commit) != 40 or any(c not in "0123456789abcdef" for c in commit): + raise SystemExit("--commit must be a full 40-hex sha that is pushed to GitHub") + + +@app.local_entrypoint() +def preflight(commit: str) -> None: + _check_commit(commit) + print(json.dumps(preflight_remote.remote(commit), indent=2)) + + +@app.local_entrypoint() +def main(commit: str, steps: str = "tests,shift,explore", script: str = "") -> None: + _check_commit(commit) + names = [s for s in steps.split(",") if s] + script_text = Path(script).read_text(encoding="utf-8") if script else "" + if script and "script" not in names: + names.append("script") + unknown = sorted(set(names) - set(STEPS) - {"script"}) + if unknown: + raise SystemExit(f"unknown steps: {unknown}") + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + commit[:9] + print(f"run_id={run_id}") + print(json.dumps(run_steps.remote(commit, names, run_id, script_text), indent=2)) + + +@app.local_entrypoint() +def launch(commit: str, steps: str = "tests,shift,explore", script: str = "") -> None: + """Spawn the run and exit at once (use with ``modal run --detach``). + + ``main`` keeps a local client attached for the whole run, and stopping that + client (for example when the machine runs low on memory) cancels the + remote input even under --detach. ``launch`` returns as soon as the call is + spawned, so nothing local can cancel it; ``collect`` fetches the results. + """ + _check_commit(commit) + names = [s for s in steps.split(",") if s] + script_text = Path(script).read_text(encoding="utf-8") if script else "" + if script and "script" not in names: + names.append("script") + unknown = sorted(set(names) - set(STEPS) - {"script"}) + if unknown: + raise SystemExit(f"unknown steps: {unknown}") + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + commit[:9] + call = run_steps.spawn(commit, names, run_id, script_text) + print(f"run_id={run_id}") + print(f"function_call_id={call.object_id}") + + +@app.local_entrypoint() +def collect(run_id: str, logs_dir: str = "") -> None: + """Write a run's changed files into this checkout and its logs to logs_dir.""" + repo = Path(__file__).resolve().parents[1] + logs = ( + Path(logs_dir) + if logs_dir + else Path.home() / ".cache" / "microcosm" / "modal-1073" / run_id + ) + logs.mkdir(parents=True, exist_ok=True) + for rel, data in fetch_results.remote(run_id).items(): + if rel.startswith("files/"): + target = repo / rel.removeprefix("files/") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + print("file", target.relative_to(repo)) + else: + (logs / rel).write_bytes(data) + print("log", logs / rel) diff --git a/experiments/receipts/1073-host-fit-diagnosis.json b/experiments/receipts/1073-host-fit-diagnosis.json new file mode 100644 index 0000000000..4251dbea75 --- /dev/null +++ b/experiments/receipts/1073-host-fit-diagnosis.json @@ -0,0 +1,960 @@ +{ + "reference_person_age_bands": { + "scf": { + "18_to_24": 0.04337107980722341, + "25_to_34": 0.15258919561765283, + "35_to_49": 0.2552374964878215, + "50_to_61": 0.1954852936381943, + "62_plus": 0.353316934449108 + }, + "cps_current": { + "18_to_24": 0.02144748678037467, + "25_to_34": 0.14294679276322914, + "35_to_49": 0.2526892646920257, + "50_to_61": 0.22104134738832001, + "62_plus": 0.3618751083760504 + }, + "cps_anchored": { + "18_to_24": 0.0339806407185135, + "25_to_34": 0.16053637335441817, + "35_to_49": 0.2644394013805101, + "50_to_61": 0.2100486920170891, + "62_plus": 0.32936379309579705 + } + }, + "rules_disagree_share": 0.13144464121075464, + "predictors_by_reference_age_current_rule": { + "18_to_24": { + "scf_married_or_partnered": 0.388466732733208, + "cps_A_MARITL_1_2": 0.3359212024332317, + "cps_spouse_or_partner_present": 0.3480042049290887, + "scf_kids_positive": 0.24031884047789556, + "cps_kids_own_of_reference_positive": 0.3147757964709492, + "cps_kids_of_reference_or_partner_positive": 0.3167990656849633, + "cps_kids_of_reference_or_partner_under_18_positive": 0.3167989510144334, + "scf_employment_income": { + "p25": 13351.351351351352, + "p50": 27729.72972972973, + "p75": 48270.27027027027, + "p90": 82162.16216216216, + "mean": 35679.59226261271 + }, + "scf_household_income": { + "p25": 21567.567567567567, + "p50": 38000.0, + "p75": 64682.16216216216, + "p90": 87297.2972972973, + "mean": 46821.884787205934 + }, + "cps_employment_income_household": { + "p25": 19942.932160142125, + "p50": 38000.0, + "p75": 65408.06916808518, + "p90": 86000.0, + "mean": 52431.37768321981 + }, + "cps_household_income_household": { + "p25": 25000.0, + "p50": 48400.0, + "p75": 81039.91162936026, + "p90": 121000.0, + "mean": 67888.96741859884 + }, + "cps_employment_income_spm_unit": { + "p25": 18043.605287747636, + "p50": 30033.106169737843, + "p75": 56000.0, + "p90": 77991.10969769866, + "mean": 46731.39114091128 + }, + "cps_household_income_spm_unit": { + "p25": 22000.0, + "p50": 45845.46664153128, + "p75": 75295.16341639643, + "p90": 103393.52709145591, + "mean": 61502.589525727824 + }, + "cps_employment_income_family": { + "p25": 18000.0, + "p50": 30000.0, + "p75": 50450.870047978584, + "p90": 70037.67841954675, + "mean": 43348.42049301801 + }, + "cps_household_income_family": { + "p25": 22000.0, + "p50": 40000.0, + "p75": 67600.0, + "p90": 88126.78110726044, + "mean": 55391.41494865657 + }, + "cps_employment_income_couple_and_children": { + "p25": 18000.0, + "p50": 30000.0, + "p75": 48400.0, + "p90": 70037.67841954675, + "mean": 43022.594188614865 + }, + "cps_household_income_couple_and_children": { + "p25": 21960.966962061266, + "p50": 38000.0, + "p75": 66990.71639962481, + "p90": 88126.78110726044, + "mean": 55065.069122927605 + }, + "cps_employment_income_couple": { + "p25": 18000.0, + "p50": 30000.0, + "p75": 48400.0, + "p90": 70037.67841954675, + "mean": 43022.56840071297 + }, + "cps_household_income_couple": { + "p25": 21960.966962061266, + "p50": 38000.0, + "p75": 66990.71639962481, + "p90": 88126.78110726044, + "mean": 55065.04193637662 + } + }, + "25_to_34": { + "scf_married_or_partnered": 0.5399894876417738, + "cps_A_MARITL_1_2": 0.4194186944982328, + "cps_spouse_or_partner_present": 0.4739143575211383, + "scf_kids_positive": 0.3846077052668725, + "cps_kids_own_of_reference_positive": 0.4335671738654255, + "cps_kids_of_reference_or_partner_positive": 0.43783228293988347, + "cps_kids_of_reference_or_partner_under_18_positive": 0.43717205052775054, + "scf_employment_income": { + "p25": 36972.97297297297, + "p50": 71891.8918918919, + "p75": 114000.0, + "p90": 174594.5945945946, + "mean": 85191.95501086062 + }, + "scf_household_income": { + "p25": 46216.21621621621, + "p50": 82162.16216216216, + "p75": 125297.29729729731, + "p90": 184864.86486486485, + "mean": 99989.46215041565 + }, + "cps_employment_income_household": { + "p25": 30033.106169737843, + "p50": 60000.0, + "p75": 108024.2158674365, + "p90": 179961.22115937775, + "mean": 83427.50903540682 + }, + "cps_household_income_household": { + "p25": 38342.35867099277, + "p50": 70000.0, + "p75": 122333.26248998212, + "p90": 198027.7607303203, + "mean": 97759.4912193431 + }, + "cps_employment_income_spm_unit": { + "p25": 29000.0, + "p50": 55000.0, + "p75": 98000.0, + "p90": 160000.0, + "mean": 76987.71046452793 + }, + "cps_household_income_spm_unit": { + "p25": 35968.50264597061, + "p50": 64407.40279386099, + "p75": 113400.0, + "p90": 183358.8295516743, + "mean": 89678.6019096458 + }, + "cps_employment_income_family": { + "p25": 25937.682601137225, + "p50": 50000.0, + "p75": 89980.61057968887, + "p90": 150046.82291916455, + "mean": 71645.64914387572 + }, + "cps_household_income_family": { + "p25": 31447.37312970231, + "p50": 59947.50440995103, + "p75": 103687.45621135348, + "p90": 168766.56367351895, + "mean": 83839.69847403152 + }, + "cps_employment_income_couple_and_children": { + "p25": 25047.37312970231, + "p50": 49976.038329879964, + "p75": 87962.57577776973, + "p90": 150046.82291916455, + "mean": 70642.49060993642 + }, + "cps_household_income_couple_and_children": { + "p25": 30034.085403312194, + "p50": 58779.602609256326, + "p75": 101685.40205932752, + "p90": 164647.15169238666, + "mean": 82573.8872383556 + }, + "cps_employment_income_couple": { + "p25": 25047.37312970231, + "p50": 49976.038329879964, + "p75": 87962.57577776973, + "p90": 150000.0, + "mean": 70591.86990128312 + }, + "cps_household_income_couple": { + "p25": 30033.106169737843, + "p50": 58779.602609256326, + "p75": 101685.40205932752, + "p90": 164647.15169238666, + "mean": 82504.91553190212 + } + }, + "35_to_49": { + "scf_married_or_partnered": 0.6290734845229035, + "cps_A_MARITL_1_2": 0.6373316031380384, + "cps_spouse_or_partner_present": 0.6612862067465197, + "scf_kids_positive": 0.6816213933375661, + "cps_kids_own_of_reference_positive": 0.6975751973361995, + "cps_kids_of_reference_or_partner_positive": 0.6991037252799116, + "cps_kids_of_reference_or_partner_under_18_positive": 0.6415944881431738, + "scf_employment_income": { + "p25": 34918.91891891892, + "p50": 82162.16216216216, + "p75": 143783.7837837838, + "p90": 225945.94594594595, + "mean": 114824.6600335461 + }, + "scf_household_income": { + "p25": 51351.35135135135, + "p50": 100648.64864864864, + "p75": 169459.45945945944, + "p90": 262364.3243243243, + "mean": 143147.9300195838 + }, + "cps_employment_income_household": { + "p25": 33475.63612595285, + "p50": 75023.41145958228, + "p75": 134100.0, + "p90": 219965.79340918665, + "mean": 107445.50962305516 + }, + "cps_household_income_household": { + "p25": 44929.96486229587, + "p50": 93715.00896244941, + "p75": 162664.02047770523, + "p90": 259851.65772947093, + "mean": 130842.88650477315 + }, + "cps_employment_income_spm_unit": { + "p25": 32419.13555318342, + "p50": 75000.0, + "p75": 131200.0, + "p90": 219965.79340918665, + "mean": 106002.14996021929 + }, + "cps_household_income_spm_unit": { + "p25": 43130.183185627066, + "p50": 91944.54097585058, + "p75": 160374.41278780956, + "p90": 258082.52630531235, + "mean": 129029.41398362057 + }, + "cps_employment_income_family": { + "p25": 30033.106169737843, + "p50": 71462.1735738426, + "p75": 130000.0, + "p90": 219016.12997298938, + "mean": 103694.39469413263 + }, + "cps_household_income_family": { + "p25": 40435.91003146334, + "p50": 90000.0, + "p75": 157623.70447324187, + "p90": 254280.3863446263, + "mean": 126358.13459474227 + }, + "cps_employment_income_couple_and_children": { + "p25": 30000.0, + "p50": 70037.67841954675, + "p75": 129985.18282949777, + "p90": 217116.8031005949, + "mean": 102415.8512812932 + }, + "cps_household_income_couple_and_children": { + "p25": 40000.0, + "p50": 89000.0, + "p75": 156473.66496267804, + "p90": 252192.08087394285, + "mean": 124568.85845079289 + }, + "cps_employment_income_couple": { + "p25": 27000.0, + "p50": 65051.94537951121, + "p75": 127000.0, + "p90": 209994.32732911556, + "mean": 98762.48810456705 + }, + "cps_household_income_couple": { + "p25": 37036.87401169252, + "p50": 83927.75612929699, + "p75": 149636.9018044752, + "p90": 243508.67594867502, + "mean": 119921.62423133879 + } + }, + "50_to_61": { + "scf_married_or_partnered": 0.5823377506882798, + "cps_A_MARITL_1_2": 0.6827162972424433, + "cps_spouse_or_partner_present": 0.6952105600998268, + "scf_kids_positive": 0.4757664892740851, + "cps_kids_own_of_reference_positive": 0.5800850452908032, + "cps_kids_of_reference_or_partner_positive": 0.5808386543289418, + "cps_kids_of_reference_or_partner_under_18_positive": 0.21670674321239264, + "scf_employment_income": { + "p25": 23621.62162162162, + "p50": 71891.8918918919, + "p75": 159189.1891891892, + "p90": 259837.83783783787, + "mean": 130259.85946328501 + }, + "scf_household_income": { + "p25": 48270.27027027027, + "p50": 100648.64864864865, + "p75": 186918.9189189189, + "p90": 328648.64864864864, + "mean": 180220.35393913506 + }, + "cps_employment_income_household": { + "p25": 40000.0, + "p50": 82900.0, + "p75": 155000.0, + "p90": 249998.89957892447, + "mean": 123769.04398414477 + }, + "cps_household_income_household": { + "p25": 63163.951453154354, + "p50": 120013.7167494267, + "p75": 199458.61673261435, + "p90": 314306.98515674786, + "mean": 164360.507613877 + }, + "cps_employment_income_spm_unit": { + "p25": 38700.0, + "p50": 81000.0, + "p75": 153014.52115728092, + "p90": 246200.0, + "mean": 122021.94149808068 + }, + "cps_household_income_spm_unit": { + "p25": 61200.0, + "p50": 117634.69211716403, + "p75": 197193.69123772596, + "p90": 312421.4000230408, + "mean": 162408.58786822122 + }, + "cps_employment_income_family": { + "p25": 37749.12158884045, + "p50": 80000.0, + "p75": 151471.31807346042, + "p90": 245036.90812479387, + "mean": 120510.2492529643 + }, + "cps_household_income_family": { + "p25": 59947.50440995103, + "p50": 114954.63160279387, + "p75": 194167.9995137999, + "p90": 312000.0, + "mean": 160553.95287265073 + }, + "cps_employment_income_couple_and_children": { + "p25": 36324.62643454458, + "p50": 77991.10969769866, + "p75": 150000.0, + "p90": 240027.4334988534, + "mean": 118613.8043169604 + }, + "cps_household_income_couple_and_children": { + "p25": 58000.0, + "p50": 113020.03074088796, + "p75": 192230.31038857266, + "p90": 310440.62512728525, + "mean": 158301.42155607024 + }, + "cps_employment_income_couple": { + "p25": 28500.0, + "p50": 65051.94537951121, + "p75": 129985.18282949776, + "p90": 216641.9713824963, + "mean": 105803.30851218582 + }, + "cps_household_income_couple": { + "p25": 47360.0, + "p50": 95849.08808532642, + "p75": 170175.8631001535, + "p90": 285000.0, + "mean": 142740.79730197633 + } + }, + "62_plus": { + "scf_married_or_partnered": 0.5111385359889865, + "cps_A_MARITL_1_2": 0.7019606119626662, + "cps_spouse_or_partner_present": 0.7078680807763289, + "scf_kids_positive": 0.15155608421924158, + "cps_kids_own_of_reference_positive": 0.29937383262058725, + "cps_kids_of_reference_or_partner_positive": 0.30075570097634075, + "cps_kids_of_reference_or_partner_under_18_positive": 0.027736868812659743, + "scf_employment_income": { + "p25": 0.0, + "p50": 0.0, + "p75": 33891.89189189189, + "p90": 103729.72972972973, + "mean": 40191.80953371061 + }, + "scf_household_income": { + "p25": 36972.97297297297, + "p50": 69837.83783783784, + "p75": 128378.37837837837, + "p90": 237243.24324324325, + "mean": 128923.04558037713 + }, + "cps_employment_income_household": { + "p25": 0.0, + "p50": 7003.7678419546755, + "p75": 60000.0, + "p90": 155000.0, + "mean": 55427.92717467862 + }, + "cps_household_income_household": { + "p25": 49926.53540083874, + "p50": 85653.87059191501, + "p75": 154569.5722100861, + "p90": 253441.3565921235, + "mean": 127766.85179782026 + }, + "cps_employment_income_spm_unit": { + "p25": 0.0, + "p50": 6000.0, + "p75": 59947.50440995103, + "p90": 151150.80666374386, + "mean": 54474.78102380924 + }, + "cps_household_income_spm_unit": { + "p25": 49150.3533963364, + "p50": 84712.14478274979, + "p75": 152507.42901065134, + "p90": 252155.32989647862, + "mean": 126486.6251861578 + }, + "cps_employment_income_family": { + "p25": 0.0, + "p50": 5000.0, + "p75": 57600.0, + "p90": 150046.82291916455, + "mean": 53421.41217379493 + }, + "cps_household_income_family": { + "p25": 48141.095029587144, + "p50": 83336.07663954931, + "p75": 150534.97679427481, + "p90": 250353.52248550823, + "mean": 125251.01837523092 + }, + "cps_employment_income_couple_and_children": { + "p25": 0.0, + "p50": 2896.473480401594, + "p75": 50000.0, + "p90": 134970.91586953326, + "mean": 47871.698899062045 + }, + "cps_household_income_couple_and_children": { + "p25": 46615.94538092046, + "p50": 80271.70886805636, + "p75": 138278.80882439725, + "p90": 231474.6077227419, + "mean": 118040.88028071057 + }, + "cps_employment_income_couple": { + "p25": 0.0, + "p50": 0.0, + "p75": 30000.0, + "p90": 99952.07665975994, + "mean": 36870.9334410469 + }, + "cps_household_income_couple": { + "p25": 39077.49835560404, + "p50": 67652.4092674109, + "p75": 118968.15700749046, + "p90": 200189.31458618637, + "mean": 103452.06921503063 + } + } + }, + "predictors_by_reference_age_anchored_rule": { + "18_to_24": { + "scf_married_or_partnered": 0.388466732733208, + "cps_A_MARITL_1_2": 0.22474301409043093, + "cps_spouse_or_partner_present": 0.24320062204068002, + "scf_kids_positive": 0.24031884047789556, + "cps_kids_own_of_reference_positive": 0.25224238041401326, + "cps_kids_of_reference_or_partner_positive": 0.2522423862320045, + "cps_kids_of_reference_or_partner_under_18_positive": 0.25224228725049114, + "scf_employment_income": { + "p25": 13351.351351351352, + "p50": 27729.72972972973, + "p75": 48270.27027027027, + "p90": 82162.16216216216, + "mean": 35679.59226261271 + }, + "scf_household_income": { + "p25": 21567.567567567567, + "p50": 38000.0, + "p75": 64682.16216216216, + "p90": 87297.2972972973, + "mean": 46821.884787205934 + }, + "cps_employment_income_household": { + "p25": 23040.0, + "p50": 46200.0, + "p75": 77991.10969769866, + "p90": 112891.24097794737, + "mean": 62177.78547112806 + }, + "cps_household_income_household": { + "p25": 31220.185464984395, + "p50": 58862.3520972485, + "p75": 88793.53128444232, + "p90": 144160.0, + "mean": 78217.66710942617 + }, + "cps_employment_income_spm_unit": { + "p25": 19942.932160142125, + "p50": 41000.0, + "p75": 70037.67841954675, + "p90": 94966.34361972439, + "mean": 55282.881554043466 + }, + "cps_household_income_spm_unit": { + "p25": 25182.020977098553, + "p50": 49026.37489368272, + "p75": 83680.04338189503, + "p90": 135237.34235554864, + "mean": 70153.52360070287 + }, + "cps_employment_income_family": { + "p25": 19942.932160142125, + "p50": 35000.0, + "p75": 59295.0, + "p90": 90000.0, + "mean": 51824.32925091325 + }, + "cps_household_income_family": { + "p25": 23160.0, + "p50": 46000.0, + "p75": 82868.14092869547, + "p90": 124999.44978946223, + "mean": 66329.47651142394 + }, + "cps_employment_income_couple_and_children": { + "p25": 14957.199120106592, + "p50": 23979.00176398041, + "p75": 44990.30528984444, + "p90": 65408.06916808518, + "mean": 36492.192696942155 + }, + "cps_household_income_couple_and_children": { + "p25": 17000.0, + "p50": 30000.0, + "p75": 54749.00186433036, + "p90": 85662.03338928835, + "mean": 46503.9521863318 + }, + "cps_employment_income_couple": { + "p25": 14957.199120106592, + "p50": 23979.00176398041, + "p75": 44990.30528984444, + "p90": 65408.06916808518, + "mean": 36492.02322061497 + }, + "cps_household_income_couple": { + "p25": 17000.0, + "p50": 30000.0, + "p75": 54749.00186433036, + "p90": 85662.03338928835, + "mean": 46503.77960773607 + } + }, + "25_to_34": { + "scf_married_or_partnered": 0.5399894876417738, + "cps_A_MARITL_1_2": 0.39715604985032005, + "cps_spouse_or_partner_present": 0.452259012174844, + "scf_kids_positive": 0.3846077052668725, + "cps_kids_own_of_reference_positive": 0.42409872327777987, + "cps_kids_of_reference_or_partner_positive": 0.4305778569704265, + "cps_kids_of_reference_or_partner_under_18_positive": 0.42916253321052183, + "scf_employment_income": { + "p25": 36972.97297297297, + "p50": 71891.8918918919, + "p75": 114000.0, + "p90": 174594.5945945946, + "mean": 85191.95501086062 + }, + "scf_household_income": { + "p25": 46216.21621621621, + "p50": 82162.16216216216, + "p75": 125297.29729729731, + "p90": 184864.86486486485, + "mean": 99989.46215041565 + }, + "cps_employment_income_household": { + "p25": 33000.0, + "p50": 62202.95507091948, + "p75": 113100.0, + "p90": 187083.69693085706, + "mean": 88380.5525425812 + }, + "cps_household_income_household": { + "p25": 41073.922849105154, + "p50": 75949.0, + "p75": 133000.0, + "p90": 219819.68333705288, + "mean": 105277.24994559937 + }, + "cps_employment_income_spm_unit": { + "p25": 30033.106169737843, + "p50": 59947.50440995103, + "p75": 104500.0, + "p90": 174975.4881193422, + "mean": 82239.9343919021 + }, + "cps_household_income_spm_unit": { + "p25": 38160.0, + "p50": 70000.0, + "p75": 121174.55918696911, + "p90": 199353.78844982432, + "mean": 97549.04089538947 + }, + "cps_employment_income_family": { + "p25": 28566.0, + "p50": 54961.77136991549, + "p75": 95000.0, + "p90": 160018.2889992356, + "mean": 76157.89229740339 + }, + "cps_household_income_family": { + "p25": 35604.32233085497, + "p50": 64355.04341526062, + "p75": 112703.91751543077, + "p90": 184869.72855043667, + "mean": 90541.35486780162 + }, + "cps_employment_income_couple_and_children": { + "p25": 25000.0, + "p50": 47958.00352796082, + "p75": 85000.0, + "p90": 148000.0, + "mean": 68543.22340338948 + }, + "cps_household_income_couple_and_children": { + "p25": 30033.106169737843, + "p50": 57454.63788993326, + "p75": 101001.0, + "p90": 161218.58424981666, + "mean": 80561.55919990591 + }, + "cps_employment_income_couple": { + "p25": 25000.0, + "p50": 47958.00352796082, + "p75": 85000.0, + "p90": 147435.24846962214, + "mean": 68492.34825035393 + }, + "cps_household_income_couple": { + "p25": 30033.106169737843, + "p50": 57335.929960408605, + "p75": 100859.32026419185, + "p90": 161218.58424981666, + "mean": 80180.74299481639 + } + }, + "35_to_49": { + "scf_married_or_partnered": 0.6290734845229035, + "cps_A_MARITL_1_2": 0.6440453902603906, + "cps_spouse_or_partner_present": 0.6653514555372678, + "scf_kids_positive": 0.6816213933375661, + "cps_kids_own_of_reference_positive": 0.7030707373318383, + "cps_kids_of_reference_or_partner_positive": 0.7049748206449906, + "cps_kids_of_reference_or_partner_under_18_positive": 0.644465986933246, + "scf_employment_income": { + "p25": 34918.91891891892, + "p50": 82162.16216216216, + "p75": 143783.7837837838, + "p90": 225945.94594594595, + "mean": 114824.6600335461 + }, + "scf_household_income": { + "p25": 51351.35135135135, + "p50": 100648.64864864864, + "p75": 169459.45945945944, + "p90": 262364.3243243243, + "mean": 143147.9300195838 + }, + "cps_employment_income_household": { + "p25": 32644.68061928026, + "p50": 75973.07489577951, + "p75": 139956.64890956884, + "p90": 220559.3330568099, + "mean": 109366.46309466714 + }, + "cps_household_income_household": { + "p25": 46409.00779119151, + "p50": 96035.69421902066, + "p75": 167809.5201741723, + "p90": 275503.0, + "mean": 135418.09577280766 + }, + "cps_employment_income_spm_unit": { + "p25": 32000.0, + "p50": 75023.41145958228, + "p75": 138057.3220371743, + "p90": 220000.0, + "mean": 108013.05830095464 + }, + "cps_household_income_spm_unit": { + "p25": 45285.13932898871, + "p50": 95000.0, + "p75": 165710.83040519131, + "p90": 272232.17023368436, + "mean": 133815.90995804372 + }, + "cps_employment_income_family": { + "p25": 30000.0, + "p50": 72767.96079861381, + "p75": 134021.25243333605, + "p90": 219965.79340918665, + "mean": 105409.16708978848 + }, + "cps_household_income_family": { + "p25": 42858.65635704456, + "p50": 92097.22979731677, + "p75": 163342.45075975615, + "p90": 269440.6022459578, + "mean": 130881.01146723755 + }, + "cps_employment_income_couple_and_children": { + "p25": 28990.0, + "p50": 70037.67841954675, + "p75": 131000.0, + "p90": 215000.0, + "mean": 102739.63122531319 + }, + "cps_household_income_couple_and_children": { + "p25": 39648.44846123493, + "p50": 89000.0, + "p75": 157340.22456503127, + "p90": 254280.3863446263, + "mean": 125446.97178857018 + }, + "cps_employment_income_couple": { + "p25": 26946.700002096797, + "p50": 65594.0, + "p75": 128000.0, + "p90": 207976.29252719643, + "mean": 99339.73185688264 + }, + "cps_household_income_couple": { + "p25": 37726.0, + "p50": 85190.84216959236, + "p75": 151739.87344439077, + "p90": 243819.0965940156, + "mean": 121211.93151601273 + } + }, + "50_to_61": { + "scf_married_or_partnered": 0.5823377506882798, + "cps_A_MARITL_1_2": 0.6976656598435952, + "cps_spouse_or_partner_present": 0.7105272865424017, + "scf_kids_positive": 0.4757664892740851, + "cps_kids_own_of_reference_positive": 0.5568859853981479, + "cps_kids_of_reference_or_partner_positive": 0.558641425893528, + "cps_kids_of_reference_or_partner_under_18_positive": 0.2240125994897446, + "scf_employment_income": { + "p25": 23621.62162162162, + "p50": 71891.8918918919, + "p75": 159189.1891891892, + "p90": 259837.83783783787, + "mean": 130259.85946328501 + }, + "scf_household_income": { + "p25": 48270.27027027027, + "p50": 100648.64864864865, + "p75": 186918.9189189189, + "p90": 328648.64864864864, + "mean": 180220.35393913506 + }, + "cps_employment_income_household": { + "p25": 38272.0, + "p50": 82900.0, + "p75": 155388.67974777403, + "p90": 252016.93438084362, + "mean": 123965.18604683633 + }, + "cps_household_income_household": { + "p25": 64188.51669456113, + "p50": 122043.97988142398, + "p75": 203218.0, + "p90": 320036.5779984712, + "mean": 166351.77819530095 + }, + "cps_employment_income_spm_unit": { + "p25": 37000.0, + "p50": 81600.0, + "p75": 155000.0, + "p90": 249998.89957892447, + "mean": 122996.81407997927 + }, + "cps_household_income_spm_unit": { + "p25": 63153.59774069108, + "p50": 120900.0, + "p75": 202318.46309585773, + "p90": 317707.0663992214, + "mean": 165210.84922738944 + }, + "cps_employment_income_family": { + "p25": 36500.0, + "p50": 80009.1444996178, + "p75": 153014.52115728095, + "p90": 249998.89957892447, + "mean": 121749.8923112994 + }, + "cps_household_income_family": { + "p25": 61000.0, + "p50": 118409.84115188659, + "p75": 200442.52502611146, + "p90": 316000.0, + "mean": 163591.7473713767 + }, + "cps_employment_income_couple_and_children": { + "p25": 36200.0, + "p50": 79000.0, + "p75": 151471.3180734604, + "p90": 245013.16653888894, + "mean": 119309.41960271333 + }, + "cps_household_income_couple_and_children": { + "p25": 58967.75203674419, + "p50": 115648.27051673843, + "p75": 194475.84052737712, + "p90": 310684.71734198555, + "mean": 159538.9537478854 + }, + "cps_employment_income_couple": { + "p25": 30000.0, + "p50": 70000.0, + "p75": 134956.83826291453, + "p90": 219965.79340918665, + "mean": 108116.09074350675 + }, + "cps_household_income_couple": { + "p25": 49775.720983738975, + "p50": 99952.07665975994, + "p75": 174635.93590561362, + "p90": 290459.1090626288, + "mean": 145711.49579543713 + } + }, + "62_plus": { + "scf_married_or_partnered": 0.5111385359889865, + "cps_A_MARITL_1_2": 0.7388492319827763, + "cps_spouse_or_partner_present": 0.7433589060264927, + "scf_kids_positive": 0.15155608421924158, + "cps_kids_own_of_reference_positive": 0.23466102562126542, + "cps_kids_of_reference_or_partner_positive": 0.23611706304714625, + "cps_kids_of_reference_or_partner_under_18_positive": 0.0280574421521517, + "scf_employment_income": { + "p25": 0.0, + "p50": 0.0, + "p75": 33891.89189189189, + "p90": 103729.72972972973, + "mean": 40191.80953371061 + }, + "scf_household_income": { + "p25": 36972.97297297297, + "p50": 69837.83783783784, + "p75": 128378.37837837837, + "p90": 237243.24324324325, + "mean": 128923.04558037713 + }, + "cps_employment_income_household": { + "p25": 0.0, + "p50": 1970.5516301092812, + "p75": 49976.038329879964, + "p90": 141025.02027529074, + "mean": 49432.38346711528 + }, + "cps_household_income_household": { + "p25": 48581.68893862247, + "p50": 82502.01101963557, + "p75": 145234.83538180616, + "p90": 241173.30773902748, + "mean": 123362.53121783666 + }, + "cps_employment_income_spm_unit": { + "p25": 0.0, + "p50": 1697.5233922025736, + "p75": 49026.37489368272, + "p90": 139956.64890956884, + "mean": 48902.609141760426 + }, + "cps_household_income_spm_unit": { + "p25": 47930.98556183415, + "p50": 82060.05538651877, + "p75": 144032.70228142804, + "p90": 239693.7450230223, + "mean": 122592.85255142464 + }, + "cps_employment_income_family": { + "p25": 0.0, + "p50": 1500.0, + "p75": 47958.00352796082, + "p90": 137345.0744600264, + "mean": 48242.34563842785 + }, + "cps_household_income_family": { + "p25": 47509.73061402481, + "p50": 81383.00936982312, + "p75": 142350.9370965536, + "p90": 238470.92633795424, + "mean": 121850.6563805075 + }, + "cps_employment_income_couple_and_children": { + "p25": 0.0, + "p50": 403.6069603838287, + "p75": 45000.0, + "p90": 131400.0, + "mean": 46279.86395045464 + }, + "cps_household_income_couple_and_children": { + "p25": 46600.19389539599, + "p50": 79475.31977810201, + "p75": 137112.48037493764, + "p90": 233097.8623748759, + "mean": 119063.15046064938 + }, + "cps_employment_income_couple": { + "p25": 0.0, + "p50": 0.0, + "p75": 30033.106169737843, + "p90": 110000.0, + "mean": 39542.742825953 + }, + "cps_household_income_couple": { + "p25": 41528.197728847954, + "p50": 71904.81898503136, + "p75": 123930.40990673419, + "p90": 210593.9002452616, + "mean": 110218.86960885866 + } + } + } +} diff --git a/experiments/receipts/1073-predictor-shift-diagnosis.json b/experiments/receipts/1073-predictor-shift-diagnosis.json new file mode 100644 index 0000000000..45ecbed3db --- /dev/null +++ b/experiments/receipts/1073-predictor-shift-diagnosis.json @@ -0,0 +1,690 @@ +{ + "distributions": { + "18_24": { + "is_female": { + "scf": { + "share": 0.41798770703096133 + }, + "cps": { + "share": 0.45308552428744164 + } + }, + "cps_race": { + "scf": { + "1": 0.4055588573289111, + "2": 0.1494317074031604, + "3": 0.12691125859473804, + "4": 0.03117349448927348, + "7": 0.2869246821839171 + }, + "cps": { + "1": 0.4912125359470461, + "2": 0.1750171196976169, + "3": 0.2643223228596663, + "4": 0.04973560604712058, + "7": 0.01971241544855015 + } + }, + "is_married": { + "scf": { + "share": 0.388466732733208 + }, + "cps": { + "share": 0.24320062204068002 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.24031884047789556, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 2.0, + "p99": 3.0 + }, + "cps": { + "share_positive": 0.2522423862320045, + "p25": 0.0, + "p50": 0.0, + "p75": 1.0, + "p90": 1.0, + "p99": 3.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.8599811794959404, + "p25": 13351.351351351352, + "p50": 27729.72972972973, + "p75": 48270.27027027027, + "p90": 82162.16216216216, + "p99": 143783.7837837838 + }, + "cps": { + "share_positive": 0.9759394993350664, + "p25": 23040.0, + "p50": 46200.0, + "p75": 77991.10969769866, + "p90": 112891.24097794737, + "p99": 252966.59781704083 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.05424986155406579, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p99": 2464.864864864865 + }, + "cps": { + "share_positive": 0.28565123900592204, + "p25": 0.0, + "p50": 0.0, + "p75": 60.0, + "p90": 1709.816556616286, + "p99": 29054.057458670093 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.024318732528120387, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p99": 21567.567567567567 + }, + "cps": { + "share_positive": 0.18299893644132223, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 30971.512260709966, + "p99": 77291.65930125325 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9838529349150277, + "p25": 21567.567567567567, + "p50": 38000.0, + "p75": 64682.16216216216, + "p90": 87297.2972972973, + "p99": 146864.86486486485 + }, + "cps": { + "share_positive": 0.999999963347847, + "p25": 31220.185464984395, + "p50": 58862.3520972485, + "p75": 88793.53128444232, + "p90": 144160.0, + "p99": 375199.535233762 + } + } + }, + "25_34": { + "is_female": { + "scf": { + "share": 0.2619838404955461 + }, + "cps": { + "share": 0.30444635032847966 + } + }, + "cps_race": { + "scf": { + "1": 0.5017149524345604, + "2": 0.14942891698567948, + "3": 0.1917846516269202, + "4": 0.03776079752055218, + "7": 0.1193106814322877 + }, + "cps": { + "1": 0.5491809062140568, + "2": 0.13721256467989754, + "3": 0.21402341380294646, + "4": 0.06370355076664147, + "7": 0.03587956453645786 + } + }, + "is_married": { + "scf": { + "share": 0.5399894876417738 + }, + "cps": { + "share": 0.4522590121748441 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.3846077052668725, + "p25": 0.0, + "p50": 0.0, + "p75": 1.0, + "p90": 2.0, + "p99": 4.0 + }, + "cps": { + "share_positive": 0.4305778569704265, + "p25": 0.0, + "p50": 0.0, + "p75": 2.0, + "p90": 2.0, + "p99": 5.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.9253064946046667, + "p25": 36972.97297297297, + "p50": 71891.8918918919, + "p75": 114000.0, + "p90": 174594.5945945946, + "p99": 269081.08108108107 + }, + "cps": { + "share_positive": 0.9557187652328238, + "p25": 33000.0, + "p50": 62202.95507091948, + "p75": 113100.0, + "p90": 187083.69693085706, + "p99": 488000.0 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.1631292191129306, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 482.70270270270265, + "p99": 5751.351351351352 + }, + "cps": { + "share_positive": 0.4197204007983822, + "p25": 0.0, + "p50": 0.0, + "p75": 829.1927011444866, + "p90": 2750.923519968109, + "p99": 35128.73801214654 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.06155435215385703, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p99": 29783.783783783783 + }, + "cps": { + "share_positive": 0.182474707324782, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 28422.962257658946, + "p99": 118514.16659422085 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9989918938108396, + "p25": 46216.21621621621, + "p50": 82162.16216216216, + "p75": 125297.29729729731, + "p90": 184864.86486486485, + "p99": 437513.5135135135 + }, + "cps": { + "share_positive": 0.994540866676791, + "p25": 41073.922849105154, + "p50": 75949.0, + "p75": 133000.0, + "p90": 219819.68333705288, + "p99": 521387.8288052998 + } + } + }, + "35_49": { + "is_female": { + "scf": { + "share": 0.2530206375476159 + }, + "cps": { + "share": 0.21152846065969416 + } + }, + "cps_race": { + "scf": { + "1": 0.5370398830446967, + "2": 0.14963993063217354, + "3": 0.14916109018056492, + "4": 0.05640742673551139, + "7": 0.10775166940705336 + }, + "cps": { + "1": 0.5781912551424031, + "2": 0.13912110647748177, + "3": 0.18770513973830344, + "4": 0.0648546948157561, + "7": 0.03012780382605546 + } + }, + "is_married": { + "scf": { + "share": 0.6290734845229035 + }, + "cps": { + "share": 0.6653514555372676 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.6816213933375661, + "p25": 0.0, + "p50": 1.0, + "p75": 2.0, + "p90": 3.0, + "p99": 5.0 + }, + "cps": { + "share_positive": 0.7049748206449906, + "p25": 0.0, + "p50": 1.0, + "p75": 2.0, + "p90": 3.0, + "p99": 5.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.886081393441535, + "p25": 34918.91891891892, + "p50": 82162.16216216216, + "p75": 143783.7837837838, + "p90": 225945.94594594595, + "p99": 688108.1081081082 + }, + "cps": { + "share_positive": 0.9362493700847381, + "p25": 32644.68061928026, + "p50": 75973.07489577951, + "p75": 139956.64890956884, + "p90": 220559.3330568099, + "p99": 799260.4894895054 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.23097723402339967, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 2290.2702702702704, + "p99": 41081.08108108108 + }, + "cps": { + "share_positive": 0.43245290547981113, + "p25": 0.0, + "p50": 0.0, + "p75": 1200.2952505810445, + "p90": 5473.829683018923, + "p99": 102204.07752230708 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.11548337010251028, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 5135.135135135135, + "p99": 54432.43243243243 + }, + "cps": { + "share_positive": 0.19991842267775162, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 34432.23815943176, + "p99": 122184.8102383467 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9975323394286034, + "p25": 51351.35135135135, + "p50": 100648.64864864864, + "p75": 169459.45945945944, + "p90": 262364.3243243243, + "p99": 862702.7027027027 + }, + "cps": { + "share_positive": 0.9923842374071725, + "p25": 46409.00779119151, + "p50": 96035.69421902066, + "p75": 167809.5201741723, + "p90": 275503.0, + "p99": 971000.0 + } + } + }, + "50_61": { + "is_female": { + "scf": { + "share": 0.2735206442912563 + }, + "cps": { + "share": 0.18106391350030282 + } + }, + "cps_race": { + "scf": { + "1": 0.6150592830522598, + "2": 0.1537290366538993, + "3": 0.12412533592366588, + "4": 0.0410681313115236, + "7": 0.06601821305865142 + }, + "cps": { + "1": 0.6537814710133548, + "2": 0.11147038885346121, + "3": 0.15250900954288946, + "4": 0.06242020394741609, + "7": 0.019818926642878383 + } + }, + "is_married": { + "scf": { + "share": 0.5823377506882798 + }, + "cps": { + "share": 0.7105272865424017 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.4757664892740851, + "p25": 0.0, + "p50": 0.0, + "p75": 1.0, + "p90": 2.0, + "p99": 4.0 + }, + "cps": { + "share_positive": 0.5586414258935282, + "p25": 0.0, + "p50": 1.0, + "p75": 2.0, + "p90": 2.0, + "p99": 4.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.8120647798629679, + "p25": 23621.62162162162, + "p50": 71891.8918918919, + "p75": 159189.1891891892, + "p90": 259837.83783783787, + "p99": 1283783.7837837837 + }, + "cps": { + "share_positive": 0.9242217192397343, + "p25": 38272.0, + "p50": 82900.0, + "p75": 155388.67974777403, + "p90": 252016.93438084362, + "p99": 893039.7538139833 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.27101727557750777, + "p25": 0.0, + "p50": 0.0, + "p75": 102.70270270270271, + "p90": 7086.486486486487, + "p99": 114308.10810810812 + }, + "cps": { + "share_positive": 0.520315648054323, + "p25": 0.0, + "p50": 50.0, + "p75": 2393.0159509939276, + "p90": 12348.937448074455, + "p99": 177569.65127567103 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.24149444109701362, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 22594.594594594593, + "p99": 108864.86486486487 + }, + "cps": { + "share_positive": 0.32165954078256986, + "p25": 0.0, + "p50": 0.0, + "p75": 14103.402241167252, + "p90": 55600.57660509128, + "p99": 157776.23506719279 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9984637035763814, + "p25": 48270.27027027027, + "p50": 100648.64864864865, + "p75": 186918.9189189189, + "p90": 328648.64864864864, + "p99": 1582648.6486486485 + }, + "cps": { + "share_positive": 0.994115224234468, + "p25": 64188.51669456113, + "p50": 122043.97988142398, + "p75": 203218.0, + "p90": 320036.5779984712, + "p99": 1040938.4747342351 + } + } + }, + "62_plus": { + "is_female": { + "scf": { + "share": 0.3270495829974353 + }, + "cps": { + "share": 0.16284687085507255 + } + }, + "cps_race": { + "scf": { + "1": 0.7215070982985176, + "2": 0.11624912623121458, + "3": 0.05977273136960041, + "4": 0.03256850555294621, + "7": 0.06990253854772117 + }, + "cps": { + "1": 0.7797796558363141, + "2": 0.07935959144499927, + "3": 0.08314726942803088, + "4": 0.04345870240384795, + "7": 0.01425478088680773 + } + }, + "is_married": { + "scf": { + "share": 0.5111385359889865 + }, + "cps": { + "share": 0.7433589060264926 + } + }, + "own_children_in_household": { + "scf": { + "share_positive": 0.15155608421924158, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 1.0, + "p99": 2.0 + }, + "cps": { + "share_positive": 0.2361170630471462, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 1.0, + "p99": 3.0 + } + }, + "employment_income": { + "scf": { + "share_positive": 0.3848493172407671, + "p25": 0.0, + "p50": 0.0, + "p75": 33891.89189189189, + "p90": 103729.72972972973, + "p99": 487837.83783783775 + }, + "cps": { + "share_positive": 0.5224639747168877, + "p25": 0.0, + "p50": 1970.5516301092812, + "p75": 49976.038329879964, + "p90": 141025.02027529074, + "p99": 600305.9996061828 + } + }, + "interest_dividend_income": { + "scf": { + "share_positive": 0.360591087221194, + "p25": 0.0, + "p50": 0.0, + "p75": 1160.5405405405406, + "p90": 15405.405405405403, + "p99": 153437.83783783784 + }, + "cps": { + "share_positive": 0.5830926043365081, + "p25": 0.0, + "p50": 437.0202506310178, + "p75": 2891.2881910941755, + "p90": 19489.630485564376, + "p99": 161045.08384893282 + } + }, + "social_security_pension_income": { + "scf": { + "share_positive": 0.8650097068004545, + "p25": 15405.405405405403, + "p50": 33891.89189189189, + "p75": 67783.78378378379, + "p90": 118108.10810810812, + "p99": 335837.8378378378 + }, + "cps": { + "share_positive": 0.7798703733427339, + "p25": 4362.970963141075, + "p50": 41057.42628607926, + "p75": 64539.98293667451, + "p90": 100348.95953445185, + "p99": 217029.24706127666 + } + }, + "household_income": { + "scf": { + "share_positive": 0.9983582259004563, + "p25": 36972.97297297297, + "p50": 69837.83783783784, + "p75": 128378.37837837837, + "p90": 237243.24324324325, + "p99": 1087621.6216216215 + }, + "cps": { + "share_positive": 0.9845537936806925, + "p25": 48581.68893862247, + "p50": 82502.01101963557, + "p75": 145234.83538180616, + "p90": 241173.30773902748, + "p99": 897701.1335746899 + } + } + } + }, + "counterfactual_shares": { + "host_as_is": { + "all": 0.14160885581858304, + "18_24": 0.2079946693400335, + "25_34": 0.26288840763041604, + "35_49": 0.20667911833738084, + "50_61": 0.12965242867557655, + "62_plus": 0.0317293089245772 + }, + "map_employment_income": { + "all": 0.14550853383577372, + "18_24": 0.20821733454837177, + "25_34": 0.2822811879547042, + "35_49": 0.22214067557292733, + "50_61": 0.12263370638995597, + "62_plus": 0.026156416929239104 + }, + "map_interest_dividend_income": { + "all": 0.15826533773859558, + "18_24": 0.25113751499207576, + "25_34": 0.29426040903559375, + "35_49": 0.2253083018522811, + "50_61": 0.15493430900783606, + "62_plus": 0.031284222225362146 + }, + "map_social_security_pension_income": { + "all": 0.14681909389719208, + "18_24": 0.25129827692417706, + "25_34": 0.2717408539776387, + "35_49": 0.2205518487572771, + "50_61": 0.1255889841224885, + "62_plus": 0.030084654967087582 + }, + "map_household_income": { + "all": 0.1511521786657064, + "18_24": 0.2519469600079321, + "25_34": 0.29107702409361075, + "35_49": 0.22018752922954252, + "50_61": 0.13158606097402478, + "62_plus": 0.030002340966301628 + }, + "map_own_children_in_household": { + "all": 0.14533313698412845, + "18_24": 0.23320659588389747, + "25_34": 0.2648754761725435, + "35_49": 0.21431410469470719, + "50_61": 0.1348310653378586, + "62_plus": 0.03003456004297946 + }, + "map_all_continuous": { + "all": 0.17321348702539888, + "18_24": 0.30505039634570374, + "25_34": 0.34426263427236925, + "35_49": 0.2618932584423557, + "50_61": 0.14637659800134883, + "62_plus": 0.02301374608038495 + } + }, + "scf_observed_shares": { + "all": 0.18154956812200357, + "18_24": 0.22183897089190358, + "25_34": 0.3615365625657711, + "35_49": 0.27186430754586627, + "50_61": 0.1666515695303606, + "62_plus": 0.041870913769168856 + } +} diff --git a/experiments/receipts/1073-recipient-definition-variants.json b/experiments/receipts/1073-recipient-definition-variants.json new file mode 100644 index 0000000000..a4cf4d0187 --- /dev/null +++ b/experiments/receipts/1073-recipient-definition-variants.json @@ -0,0 +1,1972 @@ +{ + "scf_2025": { + "share": 0.18154956812200357, + "parent_plus_share": 0.13750075307389345, + "parent_plus_by_reference_age": { + "24_and_younger": 0.0, + "25_to_34": 0.0036635524662026776, + "35_to_49": 0.041223882043333035, + "50_to_61": 0.352775474436351, + "62_and_older": 0.6050140082066444 + }, + "by_head_age_band": { + "24_and_younger": 0.22183897089190358, + "25_to_34": 0.3615365625657711, + "35_to_49": 0.27186430754586627, + "50_to_61": 0.1666515695303606, + "62_and_older": 0.041870913769168856 + }, + "by_income_quintile": { + "1": 0.1144757325207629, + "2": 0.1705804998024808, + "3": 0.17774016899539674, + "4": 0.26595688145561025, + "5": 0.1790121945626286 + } + }, + "variants": { + "v0_first_receipt": { + "definition": { + "reference_person": "oldest_adult", + "partnered": "married_spouse", + "children": "of_reference", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060221, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "The first receipt's construction.", + "share": 0.14412999386354491, + "parent_plus_share": 0.21495679635292875, + "parent_plus_by_reference_age": { + "24_and_younger": 0.05361481308447381, + "25_to_34": 0.00022032123488229902, + "35_to_49": 0.03526511572758998, + "50_to_61": 0.43355500134530767, + "62_and_older": 0.7000193215077478 + }, + "by_head_age_band": { + "24_and_younger": 0.2267174320421409, + "25_to_34": 0.2669429791293293, + "35_to_49": 0.22339688829964577, + "50_to_61": 0.14218578094851128, + "62_and_older": 0.03655929815086838 + }, + "by_income_quintile": { + "1": 0.12721563759800417, + "2": 0.1358771691149805, + "3": 0.13382646407799811, + "4": 0.19993030155269514, + "5": 0.12378308052696536 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7419574258458654, + "mean_among_borrowers_error_pct": 8.357217142313411, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.061856909978440955, + "head_age_band_share_mae_pp": 3.5543373606609885, + "race_share_mae_pp": 2.8343426729118955, + "income_quintile_share_mae_pp": 4.2522526924143715, + "borrower_balance_band_tvd": 0.06971797938468582, + "borrower_age_band_tvd": 0.08466988479278444 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14412999386354491, + "mean_among_borrowers": 44483.09961323457, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 785628046119.6727, + "parent_plus_share_of_total": 0.21495679635292875 + }, + "household_balance_bands": { + "under_5k": 0.09597901426395242, + "5k_to_10k": 0.1620676904114682, + "10k_to_20k": 0.15178493618645325, + "20k_to_40k": 0.22796648343692355, + "40k_to_60k": 0.09915026443778263, + "60k_to_80k": 0.10849192930212849, + "80k_to_100k": 0.056043492646084624, + "100k_to_200k": 0.06743544596300373, + "200k_plus": 0.031080743352203163 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.2267174320421409, + "mean_among_borrowers": 18670.63953159211, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2669429791293293, + "mean_among_borrowers": 36447.86184035785, + "median_among_borrowers": 20200.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22339688829964577, + "mean_among_borrowers": 44315.447461546944, + "median_among_borrowers": 25670.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14218578094851128, + "mean_among_borrowers": 49552.354866035355, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03655929815086838, + "mean_among_borrowers": 65818.86253190483, + "median_among_borrowers": 38000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13333034770117472, + "mean_among_borrowers": 42577.59434933696, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902667788102, + "mean_among_borrowers": 47993.699863516216, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12291729609744638, + "mean_among_borrowers": 46285.94561484015, + "median_among_borrowers": 24764.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15337638725400463, + "mean_among_borrowers": 47893.571558628166, + "median_among_borrowers": 24000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12721563759800417, + "mean_among_borrowers": 38171.39283093021, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1358771691149805, + "mean_among_borrowers": 45461.58729969715, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13382646407799811, + "mean_among_borrowers": 40922.60303950533, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19993030155269514, + "mean_among_borrowers": 45004.488576349744, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12378308052696536, + "mean_among_borrowers": 52902.03665110157, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18461703.3923496, + "mean_balance": 42554.4723270352, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.1039144177980828, + "5k_to_10k": 0.16665917220324747, + "10k_to_20k": 0.15515901493993045, + "20k_to_40k": 0.23029463650667883, + "40k_to_60k": 0.10426754245310213, + "60k_to_80k": 0.09578243033185131, + "80k_to_100k": 0.05074417014558102, + "100k_to_200k": 0.06376833884555547, + "200k_plus": 0.029410276775970686 + }, + "age_bands": { + "24_and_younger": 0.03769253987750687, + "25_to_34": 0.2662405413523508, + "35_to_49": 0.39817213354338704, + "50_to_61": 0.21071813444930676, + "62_and_older": 0.08717665077744856 + }, + "graduate_share_of_own_original_principal": 0.5360724443816975, + "ibr_new_borrower_share": 0.605734686749079 + } + } + }, + "v1_partner_counts_as_married": { + "definition": { + "reference_person": "oldest_adult", + "partnered": "spouse_or_partner", + "children": "of_reference", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060221, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "SCF MARRIED is married or living with a partner (X8023 in 1-2); A_MARITL 1-2 drops cohabiting partners.", + "share": 0.14421786795177846, + "parent_plus_share": 0.2383545964760273, + "parent_plus_by_reference_age": { + "24_and_younger": 2.253143474595712e-05, + "25_to_34": 0.0031437172991370397, + "35_to_49": 0.03346246118041894, + "50_to_61": 0.5087392931361827, + "62_and_older": 0.7514633083579593 + }, + "by_head_age_band": { + "24_and_younger": 0.22671795358234773, + "25_to_34": 0.2681773827190304, + "35_to_49": 0.22322300770469652, + "50_to_61": 0.14171355085128, + "62_and_older": 0.03672435234678404 + }, + "by_income_quintile": { + "1": 0.1272156431154305, + "2": 0.1359920694211529, + "3": 0.13490203218158234, + "4": 0.19906487838581546, + "5": 0.12389784553658897 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.733170017022511, + "mean_among_borrowers_error_pct": 8.963130475592095, + "median_among_borrowers_error_pct": 1.200000000000001, + "household_balance_band_tvd": 0.06098877733974077, + "head_age_band_share_mae_pp": 3.5392808495963997, + "race_share_mae_pp": 2.895639943104994, + "income_quintile_share_mae_pp": 4.2434565977128855, + "borrower_balance_band_tvd": 0.06591147421877885, + "borrower_age_band_tvd": 0.07517303040860317 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14421786795177846, + "mean_among_borrowers": 44731.84080345749, + "median_among_borrowers": 25300.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 790502796606.3756, + "parent_plus_share_of_total": 0.2383545964760273 + }, + "household_balance_bands": { + "under_5k": 0.09724840853122779, + "5k_to_10k": 0.15964535820912895, + "10k_to_20k": 0.14731451015137764, + "20k_to_40k": 0.23069297158741, + "40k_to_60k": 0.10134840208870029, + "60k_to_80k": 0.10851424309963001, + "80k_to_100k": 0.05607431201281855, + "100k_to_200k": 0.06747102087873265, + "200k_plus": 0.03169077344097403 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.22671795358234773, + "mean_among_borrowers": 18861.163422392947, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2681773827190304, + "mean_among_borrowers": 37146.8559849801, + "median_among_borrowers": 22000.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22322300770469652, + "mean_among_borrowers": 44393.29221805572, + "median_among_borrowers": 25944.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14171355085128, + "mean_among_borrowers": 49673.12225637845, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03672435234678404, + "mean_among_borrowers": 65867.18117871678, + "median_among_borrowers": 39000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13395046003400674, + "mean_among_borrowers": 42718.245961511835, + "median_among_borrowers": 25300.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902879591022, + "mean_among_borrowers": 48460.870926229596, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12189634611539013, + "mean_among_borrowers": 46674.41390252431, + "median_among_borrowers": 25000.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15132533197747572, + "mean_among_borrowers": 48377.79858657893, + "median_among_borrowers": 24700.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.1272156431154305, + "mean_among_borrowers": 38691.57607287244, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1359920694211529, + "mean_among_borrowers": 45423.29084841887, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13490203218158234, + "mean_among_borrowers": 41268.331115616085, + "median_among_borrowers": 25300.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19906487838581546, + "mean_among_borrowers": 45324.769240708556, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12389784553658897, + "mean_among_borrowers": 52992.43425311495, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18499654.177020565, + "mean_balance": 42730.679667963865, + "median_balance": 24321.0, + "balance_bands": { + "under_5k": 0.10864552035546077, + "5k_to_10k": 0.16261363075134405, + "10k_to_20k": 0.155785250662947, + "20k_to_40k": 0.22691872382328185, + "40k_to_60k": 0.10271670933861446, + "60k_to_80k": 0.10059174421550436, + "80k_to_100k": 0.04980040443960724, + "100k_to_200k": 0.06284484580513604, + "200k_plus": 0.03008317060810413 + }, + "age_bands": { + "24_and_younger": 0.039091157227139205, + "25_to_34": 0.27433877838689963, + "35_to_49": 0.3897180634972089, + "50_to_61": 0.20575528706790563, + "62_and_older": 0.09109671382084646 + }, + "graduate_share_of_own_original_principal": 0.5381213307355639, + "ibr_new_borrower_share": 0.6057402495033247 + } + } + }, + "v2_children_of_reference_or_partner": { + "definition": { + "reference_person": "oldest_adult", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060221, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "SCF KIDS counts children of the reference person or spouse/partner, any age, anywhere in the household listing.", + "share": 0.1394955127190334, + "parent_plus_share": 0.1797625275395828, + "parent_plus_by_reference_age": { + "24_and_younger": 1.8950713199477208e-09, + "25_to_34": 0.003974829123535003, + "35_to_49": 0.023750250056890892, + "50_to_61": 0.4267091997855418, + "62_and_older": 0.6189997224359128 + }, + "by_head_age_band": { + "24_and_younger": 0.23591005978245044, + "25_to_34": 0.2765552999001228, + "35_to_49": 0.21374384005656138, + "50_to_61": 0.13109688553788776, + "62_and_older": 0.03292444110763649 + }, + "by_income_quintile": { + "1": 0.12259529843027182, + "2": 0.13647608213109577, + "3": 0.1390432941963818, + "4": 0.1883195761175596, + "5": 0.11102922220126195 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -4.205405540297017, + "mean_among_borrowers_error_pct": 5.248358030394584, + "median_among_borrowers_error_pct": 1.200000000000001, + "household_balance_band_tvd": 0.04583400383836792, + "head_age_band_share_mae_pp": 4.033479513990105, + "race_share_mae_pp": 3.346280982970381, + "income_quintile_share_mae_pp": 4.5308227215865235, + "borrower_balance_band_tvd": 0.05793805362184862, + "borrower_age_band_tvd": 0.05544035124223032 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.1394955127190334, + "mean_among_borrowers": 43206.84231163398, + "median_among_borrowers": 25300.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 738550734853.654, + "parent_plus_share_of_total": 0.1797625275395828 + }, + "household_balance_bands": { + "under_5k": 0.09939228853825689, + "5k_to_10k": 0.1535814556424139, + "10k_to_20k": 0.15338979008804154, + "20k_to_40k": 0.23417095609696748, + "40k_to_60k": 0.10944941047868599, + "60k_to_80k": 0.10279164254586112, + "80k_to_100k": 0.04997604706138593, + "100k_to_200k": 0.06911469871793441, + "200k_plus": 0.028133710830452702 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.23591005978245044, + "mean_among_borrowers": 21290.46116231051, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2765552999001228, + "mean_among_borrowers": 36585.71819351528, + "median_among_borrowers": 21600.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.21374384005656138, + "mean_among_borrowers": 44071.0450196076, + "median_among_borrowers": 26500.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.13109688553788776, + "mean_among_borrowers": 49220.30130728327, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03292444110763649, + "mean_among_borrowers": 55939.813048540644, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.1289552997263875, + "mean_among_borrowers": 41158.906258343646, + "median_among_borrowers": 25300.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22416810893631395, + "mean_among_borrowers": 47150.30266408629, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.11808625225094552, + "mean_among_borrowers": 44282.98610623642, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.14618586441452036, + "mean_among_borrowers": 47779.9480237213, + "median_among_borrowers": 24700.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12259529843027182, + "mean_among_borrowers": 38343.347009388315, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.13647608213109577, + "mean_among_borrowers": 45292.80429539413, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.1390432941963818, + "mean_among_borrowers": 38379.766451262985, + "median_among_borrowers": 25000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.1883195761175596, + "mean_among_borrowers": 43996.9806856208, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.11102922220126195, + "mean_among_borrowers": 50716.140548307994, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 17825507.124679778, + "mean_balance": 41432.242554890094, + "median_balance": 25000.0, + "balance_bands": { + "under_5k": 0.10587106866451161, + "5k_to_10k": 0.15716435659164057, + "10k_to_20k": 0.15459448173794518, + "20k_to_40k": 0.23795597452795925, + "40k_to_60k": 0.11465535055149564, + "60k_to_80k": 0.09453438820382777, + "80k_to_100k": 0.043584550860481554, + "100k_to_200k": 0.06582616567410109, + "200k_plus": 0.025813663188037198 + }, + "age_bands": { + "24_and_younger": 0.04174644796694323, + "25_to_34": 0.2914161668134685, + "35_to_49": 0.383747423990049, + "50_to_61": 0.1994666870240408, + "62_and_older": 0.08362327420549835 + }, + "graduate_share_of_own_original_principal": 0.5133189129620963, + "ibr_new_borrower_share": 0.5894419667702057 + } + } + }, + "v3_householder_couple_reference_person": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060221, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "The SCF reference person comes from the economically dominant individual or couple (the man, or the older partner); the CPS householder anchors that unit, where the oldest adult can be a co-resident parent or roommate.", + "share": 0.14160885581858304, + "parent_plus_share": 0.16635099149589064, + "parent_plus_by_reference_age": { + "24_and_younger": 0.00017157399333482214, + "25_to_34": 0.005150069210217926, + "35_to_49": 0.02057959141870075, + "50_to_61": 0.4290184374286386, + "62_and_older": 0.6324187709904693 + }, + "by_head_age_band": { + "24_and_younger": 0.19846806978919576, + "25_to_34": 0.26288840763041604, + "35_to_49": 0.20667911833738084, + "50_to_61": 0.12965242867557655, + "62_and_older": 0.0317293089245772 + }, + "by_income_quintile": { + "1": 0.1262566286047617, + "2": 0.14177829750298432, + "3": 0.1357961209373581, + "4": 0.19272855509869874, + "5": 0.11146869978027 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.994071230342053, + "mean_among_borrowers_error_pct": 3.531511434829504, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.03457855760861366, + "head_age_band_share_mae_pp": 4.686899818918479, + "race_share_mae_pp": 2.880989091350316, + "income_quintile_share_mae_pp": 4.46597935161608, + "borrower_balance_band_tvd": 0.048461501202019044, + "borrower_age_band_tvd": 0.027331980297340144 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14160885581858304, + "mean_among_borrowers": 42502.03777581002, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 737509698707.2527, + "parent_plus_share_of_total": 0.16635099149589064 + }, + "household_balance_bands": { + "under_5k": 0.09993330421288438, + "5k_to_10k": 0.1508675449396742, + "10k_to_20k": 0.16976059764222085, + "20k_to_40k": 0.2275283363657654, + "40k_to_60k": 0.11015794699391307, + "60k_to_80k": 0.09824782711371129, + "80k_to_100k": 0.046130984805146207, + "100k_to_200k": 0.0690932668947163, + "200k_plus": 0.028280191031968407 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.19846806978919576, + "mean_among_borrowers": 19571.066784291823, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.26288840763041604, + "mean_among_borrowers": 38450.03383676039, + "median_among_borrowers": 21000.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.20667911833738084, + "mean_among_borrowers": 42931.01574498944, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.12965242867557655, + "mean_among_borrowers": 47692.45531862902, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.0317293089245772, + "mean_among_borrowers": 58604.73232987876, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.12977996862923966, + "mean_among_borrowers": 41370.06905388904, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.2294764658429962, + "mean_among_borrowers": 45845.065773264745, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11656658232690145, + "mean_among_borrowers": 42058.17041201836, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.16018418419383262, + "mean_among_borrowers": 43738.317358090426, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.1262566286047617, + "mean_among_borrowers": 38482.73985057685, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.14177829750298432, + "mean_among_borrowers": 42757.013143750206, + "median_among_borrowers": 25000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.1357961209373581, + "mean_among_borrowers": 36845.744104474776, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19272855509869874, + "mean_among_borrowers": 42892.848389119885, + "median_among_borrowers": 25629.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.11146869978027, + "mean_among_borrowers": 52943.922322276216, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18148489.81162718, + "mean_balance": 40637.52446414318, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.11021695554879672, + "5k_to_10k": 0.152438368597162, + "10k_to_20k": 0.16997945471178874, + "20k_to_40k": 0.23626211817475332, + "40k_to_60k": 0.10727311904109983, + "60k_to_80k": 0.09305715931666757, + "80k_to_100k": 0.03975383525974363, + "100k_to_200k": 0.06463705675017781, + "200k_plus": 0.026381932599810472 + }, + "age_bands": { + "24_and_younger": 0.05449396163740877, + "25_to_34": 0.30902579405669556, + "35_to_49": 0.3785313140100526, + "50_to_61": 0.18776495318129746, + "62_and_older": 0.07018397711454578 + }, + "graduate_share_of_own_original_principal": 0.519651843154687, + "ibr_new_borrower_share": 0.609045516567325 + } + } + }, + "v4a_spm_unit_income": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "spm_unit" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060221, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "SCF income covers the primary economic unit; the SPM unit is the CPS resource-sharing unit.", + "share": 0.1422480075434781, + "parent_plus_share": 0.1792457232848817, + "parent_plus_by_reference_age": { + "24_and_younger": 2.0364004798094018e-08, + "25_to_34": 0.0035008167958351234, + "35_to_49": 0.029458562221742368, + "50_to_61": 0.47869069913423207, + "62_and_older": 0.6254994464664674 + }, + "by_head_age_band": { + "24_and_younger": 0.19738286662117155, + "25_to_34": 0.26409298361810984, + "35_to_49": 0.2083709419645182, + "50_to_61": 0.1302801328964551, + "62_and_older": 0.031441439273676396 + }, + "by_income_quintile": { + "1": 0.12941404718889285, + "2": 0.14024648681888457, + "3": 0.13552164034883177, + "4": 0.193981131880552, + "5": 0.11208227675067776 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.930156057852546, + "mean_among_borrowers_error_pct": 3.5620052430742177, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.03742705681897808, + "head_age_band_share_mae_pp": 4.643879198582787, + "race_share_mae_pp": 2.863735982595237, + "income_quintile_share_mae_pp": 4.527930473706005, + "borrower_balance_band_tvd": 0.039884034926986736, + "borrower_age_band_tvd": 0.035403180196326975 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.1422480075434781, + "mean_among_borrowers": 42514.55617694204, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 741056653493.6942, + "parent_plus_share_of_total": 0.1792457232848817 + }, + "household_balance_bands": { + "under_5k": 0.09952116824675251, + "5k_to_10k": 0.15435171448733467, + "10k_to_20k": 0.16562804147553756, + "20k_to_40k": 0.22674020689102248, + "40k_to_60k": 0.11279034359745732, + "60k_to_80k": 0.09923618460456847, + "80k_to_100k": 0.04557077463266375, + "100k_to_200k": 0.0678800899029161, + "200k_plus": 0.028281476161747247 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.19738286662117155, + "mean_among_borrowers": 19442.015941146798, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.26409298361810984, + "mean_among_borrowers": 38436.58028829374, + "median_among_borrowers": 20500.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.2083709419645182, + "mean_among_borrowers": 42853.01799862837, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.1302801328964551, + "mean_among_borrowers": 47861.768338455506, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.031441439273676396, + "mean_among_borrowers": 58939.872552347515, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.13097596521508803, + "mean_among_borrowers": 41202.78031031787, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.22831695407527614, + "mean_among_borrowers": 45683.93405244018, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11574381829957685, + "mean_among_borrowers": 41667.139357186345, + "median_among_borrowers": 22000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.16166058775323208, + "mean_among_borrowers": 45928.990341343546, + "median_among_borrowers": 25000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24498252.26087785, + "share_with_federal_education_debt": 0.12941404718889285, + "mean_among_borrowers": 38621.5541253665, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24516086.12746574, + "share_with_federal_education_debt": 0.14024648681888457, + "mean_among_borrowers": 43623.80714350449, + "median_among_borrowers": 26087.0 + }, + "3": { + "household_weight": 24508573.098401047, + "share_with_federal_education_debt": 0.13552164034883177, + "mean_among_borrowers": 35795.30093902187, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 24504387.547067694, + "share_with_federal_education_debt": 0.193981131880552, + "mean_among_borrowers": 41327.58298000465, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24509792.851870958, + "share_with_federal_education_debt": 0.11208227675067776, + "mean_among_borrowers": 55796.96154188077, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18198229.29909551, + "mean_balance": 40721.360376007906, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.10990097377367777, + "5k_to_10k": 0.15597631321107155, + "10k_to_20k": 0.1732232936496721, + "20k_to_40k": 0.22454924901870585, + "40k_to_60k": 0.11005065222546452, + "60k_to_80k": 0.09520726630411006, + "80k_to_100k": 0.04158394034849486, + "100k_to_200k": 0.0635499527510756, + "200k_plus": 0.025958358717727704 + }, + "age_bands": { + "24_and_younger": 0.0529887481725192, + "25_to_34": 0.3002110376537959, + "35_to_49": 0.3829114048942107, + "50_to_61": 0.19230144811764502, + "62_and_older": 0.07158736116182915 + }, + "graduate_share_of_own_original_principal": 0.5365614143034969, + "ibr_new_borrower_share": 0.6249721194428659 + } + } + }, + "v4b_couple_and_children_income": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17281627661600854, + "parent_plus_share": 0.11558498718060221, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.349507836301458, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.037246827952770664, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.030251768076485816, + "borrower_age_band_tvd": 0.008733492308356343 + } + }, + "reason": "The PEU narrowed to the couple and their children.", + "share": 0.14453729401731658, + "parent_plus_share": 0.15519605817424037, + "parent_plus_by_reference_age": { + "24_and_younger": 1.352905945393419e-08, + "25_to_34": 0.00013998491463271278, + "35_to_49": 0.019468440588935945, + "50_to_61": 0.3913612095514226, + "62_and_older": 0.5990857762189746 + }, + "by_head_age_band": { + "24_and_younger": 0.2109485663269708, + "25_to_34": 0.2709152074679265, + "35_to_49": 0.20910334189997756, + "50_to_61": 0.1331061797690564, + "62_and_older": 0.03120975131828736 + }, + "by_income_quintile": { + "1": 0.13612388616760965, + "2": 0.1459793402381904, + "3": 0.13033686972701333, + "4": 0.1919836615881704, + "5": 0.11826299055685831 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7012274104686993, + "mean_among_borrowers_error_pct": 2.7320841095266557, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.040156439181474324, + "head_age_band_share_mae_pp": 4.169585550417035, + "race_share_mae_pp": 2.569878492307957, + "income_quintile_share_mae_pp": 4.567500727054615, + "borrower_balance_band_tvd": 0.04019409665288553, + "borrower_age_band_tvd": 0.03235991026200852 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14453729401731658, + "mean_among_borrowers": 42173.85469504409, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 746948718213.9073, + "parent_plus_share_of_total": 0.15519605817424037 + }, + "household_balance_bands": { + "under_5k": 0.10775015906187184, + "5k_to_10k": 0.1528994221663368, + "10k_to_20k": 0.15878382717932804, + "20k_to_40k": 0.23707174823778418, + "40k_to_60k": 0.11254827709408233, + "60k_to_80k": 0.09506114425698398, + "80k_to_100k": 0.044996423198641146, + "100k_to_200k": 0.05939892961997767, + "200k_plus": 0.03149006918499395 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.2109485663269708, + "mean_among_borrowers": 22164.502413760794, + "median_among_borrowers": 14000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.2709152074679265, + "mean_among_borrowers": 36647.47528338518, + "median_among_borrowers": 21600.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.20910334189997756, + "mean_among_borrowers": 42586.38683331958, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.1331061797690564, + "mean_among_borrowers": 49336.50631455528, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.03120975131828736, + "mean_among_borrowers": 58478.07907213402, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.13312784049029713, + "mean_among_borrowers": 40718.755628997955, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.22788477777157826, + "mean_among_borrowers": 47824.23980122162, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11755065774394279, + "mean_among_borrowers": 39599.36261332915, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.1698883489488461, + "mean_among_borrowers": 44027.69290376679, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.13612388616760965, + "mean_among_borrowers": 38386.48367756073, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.1459793402381904, + "mean_among_borrowers": 42114.80663102074, + "median_among_borrowers": 23000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.13033686972701333, + "mean_among_borrowers": 39083.07983863153, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.1919836615881704, + "mean_among_borrowers": 42068.527873712716, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.11826299055685831, + "mean_among_borrowers": 50185.22911179778, + "median_among_borrowers": 26814.0 + } + }, + "borrowers": { + "borrower_weight": 18531371.849430196, + "mean_balance": 40307.25432973677, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.11848284445361304, + "5k_to_10k": 0.15391028318710992, + "10k_to_20k": 0.16124932842055162, + "20k_to_40k": 0.23782638366100617, + "40k_to_60k": 0.11453284638670623, + "60k_to_80k": 0.09065262966356538, + "80k_to_100k": 0.0390975463353386, + "100k_to_200k": 0.05475398408033307, + "200k_plus": 0.029494153811776037 + }, + "age_bands": { + "24_and_younger": 0.05515156793888616, + "25_to_34": 0.3089032338721271, + "35_to_49": 0.3729683379284752, + "50_to_61": 0.19389979448680775, + "62_and_older": 0.06907706577370382 + }, + "graduate_share_of_own_original_principal": 0.4975010554094781, + "ibr_new_borrower_share": 0.6165370510095016 + } + } + }, + "r1_couple_income_drop_interest_dividends": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "social_security_pension_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "social_security_pension_income", + "household_income" + ], + "share": 0.17429665336106148, + "parent_plus_share": 0.12799128561614068, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.7252914760942086, + "mean_among_borrowers_error_pct": -6.467428746293713, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03359502345865293, + "head_age_band_share_mae_pp": 1.1373174155641717, + "race_share_mae_pp": 1.2886634772442556, + "income_quintile_share_mae_pp": 0.8724139723993973, + "borrower_balance_band_tvd": 0.03705671769663749, + "borrower_age_band_tvd": 0.013839814547848696 + } + }, + "reason": "CPS interest and dividends are PUF-imputed at tax-unit level; at reference ages 25-34, 42% of host households have some against 16% of SCF families, so equal values do not mean equal positions.", + "share": 0.16068368510572498, + "parent_plus_share": 0.16139644121762028, + "parent_plus_by_reference_age": { + "24_and_younger": 4.3569134442118664e-10, + "25_to_34": 0.0006982323042478812, + "35_to_49": 0.016124991871770737, + "50_to_61": 0.37078829780154465, + "62_and_older": 0.7327689853082328 + }, + "by_head_age_band": { + "24_and_younger": 0.22636975285526706, + "25_to_34": 0.28177775577348513, + "35_to_49": 0.23675283607454572, + "50_to_61": 0.16663378760977693, + "62_and_older": 0.029689656574013047 + }, + "by_income_quintile": { + "1": 0.14148727103894412, + "2": 0.14782938572653392, + "3": 0.14763888154405314, + "4": 0.2260334704473883, + "5": 0.1404236189948884 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -2.086588301627859, + "mean_among_borrowers_error_pct": 5.419007269798315, + "median_among_borrowers_error_pct": -1.200000000000001, + "household_balance_band_tvd": 0.03897139087869254, + "head_age_band_share_mae_pp": 2.6320019868541893, + "race_share_mae_pp": 1.715677598047579, + "income_quintile_share_mae_pp": 3.1675185324286774, + "borrower_balance_band_tvd": 0.03809557472261625, + "borrower_age_band_tvd": 0.05104758904754849 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16068368510572498, + "mean_among_borrowers": 43276.897701717964, + "median_among_borrowers": 24700.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 852109629770.2631, + "parent_plus_share_of_total": 0.16139644121762028 + }, + "household_balance_bands": { + "under_5k": 0.10449852122435789, + "5k_to_10k": 0.15065860397660183, + "10k_to_20k": 0.16882661644138566, + "20k_to_40k": 0.23081137027456972, + "40k_to_60k": 0.10882288460069202, + "60k_to_80k": 0.09579589273732184, + "80k_to_100k": 0.04072416122031918, + "100k_to_200k": 0.06714054669721696, + "200k_plus": 0.03272140282753487 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.22636975285526706, + "mean_among_borrowers": 20232.49190929045, + "median_among_borrowers": 7940.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28177775577348513, + "mean_among_borrowers": 35902.8761787989, + "median_among_borrowers": 22000.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.23675283607454572, + "mean_among_borrowers": 45150.70392260933, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16663378760977693, + "mean_among_borrowers": 50437.639301615825, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.029689656574013047, + "mean_among_borrowers": 58758.63181213605, + "median_among_borrowers": 40000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.14966927917838, + "mean_among_borrowers": 42682.42788830105, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.25697435771393307, + "mean_among_borrowers": 46819.536501712115, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.12815612550195352, + "mean_among_borrowers": 36458.037818012526, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.1749032891253265, + "mean_among_borrowers": 49540.69039492447, + "median_among_borrowers": 21000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14148727103894412, + "mean_among_borrowers": 42475.669637712905, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.14782938572653392, + "mean_among_borrowers": 41544.88128808425, + "median_among_borrowers": 21997.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14763888154405314, + "mean_among_borrowers": 39417.16786016642, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.2260334704473883, + "mean_among_borrowers": 41212.809485576174, + "median_among_borrowers": 27000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1404236189948884, + "mean_among_borrowers": 53288.26313186684, + "median_among_borrowers": 25000.0 + } + }, + "borrowers": { + "borrower_weight": 20417273.566150248, + "mean_balance": 41734.74127235938, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.11809168217956414, + "5k_to_10k": 0.15130000841757468, + "10k_to_20k": 0.16989131376762578, + "20k_to_40k": 0.2286785190554563, + "40k_to_60k": 0.11099082001348541, + "60k_to_80k": 0.08837266814620766, + "80k_to_100k": 0.03787968228128524, + "100k_to_200k": 0.06370360539613464, + "200k_plus": 0.031091700742666088 + }, + "age_bands": { + "24_and_younger": 0.05398123485212966, + "25_to_34": 0.29133756126405175, + "35_to_49": 0.3828432273021299, + "50_to_61": 0.2105771421991523, + "62_and_older": 0.06126083438253632 + }, + "graduate_share_of_own_original_principal": 0.49921320972671657, + "ibr_new_borrower_share": 0.5832347045593139 + } + } + }, + "r2_couple_income_drop_interest_dividends_and_ss_pension": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "share": 0.1785134113597766, + "parent_plus_share": 0.12228737578012734, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.30361567622269603, + "mean_among_borrowers_error_pct": -7.119030826113226, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03158207267342461, + "head_age_band_share_mae_pp": 1.2596405967540016, + "race_share_mae_pp": 1.069077118976515, + "income_quintile_share_mae_pp": 0.6750791601254419, + "borrower_balance_band_tvd": 0.032948871945896414, + "borrower_age_band_tvd": 0.014803476410333236 + } + }, + "reason": "As r1, and Social Security or pension income: 18% of host households at 25-34 have some against 6% of SCF families.", + "share": 0.16413671000206656, + "parent_plus_share": 0.16758194869071996, + "parent_plus_by_reference_age": { + "24_and_younger": 0.0009290910285421434, + "25_to_34": 0.003457206986791982, + "35_to_49": 0.02044429126371113, + "50_to_61": 0.317410103256937, + "62_and_older": 0.7753235949011145 + }, + "by_head_age_band": { + "24_and_younger": 0.21797195752357604, + "25_to_34": 0.28107974022687005, + "35_to_49": 0.24429588401566177, + "50_to_61": 0.16352578267462928, + "62_and_older": 0.037347739302272136 + }, + "by_income_quintile": { + "1": 0.14257834053880739, + "2": 0.15791199105287693, + "3": 0.14719109419650156, + "4": 0.23072142160681058, + "5": 0.1422795004455311 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.7412858119937007, + "mean_among_borrowers_error_pct": 3.965667719599386, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.046105502753231414, + "head_age_band_share_mae_pp": 2.3908244112012222, + "race_share_mae_pp": 0.9415277758683394, + "income_quintile_share_mae_pp": 2.865766910648814, + "borrower_balance_band_tvd": 0.04473225112436991, + "borrower_age_band_tvd": 0.049566199767822304 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16413671000206656, + "mean_among_borrowers": 42680.26879514089, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 858421208954.9309, + "parent_plus_share_of_total": 0.16758194869071996 + }, + "household_balance_bands": { + "under_5k": 0.1179407532730876, + "5k_to_10k": 0.1324473758996882, + "10k_to_20k": 0.1827494270561508, + "20k_to_40k": 0.23417620772637385, + "40k_to_60k": 0.10433535383061796, + "60k_to_80k": 0.0976346772652531, + "80k_to_100k": 0.0379014108379451, + "100k_to_200k": 0.05781213394341271, + "200k_plus": 0.03500266016747067 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.21797195752357604, + "mean_among_borrowers": 20475.641978208863, + "median_among_borrowers": 12662.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28107974022687005, + "mean_among_borrowers": 34573.95711939027, + "median_among_borrowers": 20100.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.24429588401566177, + "mean_among_borrowers": 44478.05328490035, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16352578267462928, + "mean_among_borrowers": 48394.24067058906, + "median_among_borrowers": 28742.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.037347739302272136, + "mean_among_borrowers": 61031.6733341336, + "median_among_borrowers": 34000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15291016385941802, + "mean_among_borrowers": 42266.19101485241, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.24861685783459786, + "mean_among_borrowers": 45500.64480289935, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.13835157436352993, + "mean_among_borrowers": 37031.934121143146, + "median_among_borrowers": 19200.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.18407544859054642, + "mean_among_borrowers": 48161.13603160578, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14257834053880739, + "mean_among_borrowers": 43328.84042017116, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.15791199105287693, + "mean_among_borrowers": 39522.35712436794, + "median_among_borrowers": 20000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14719109419650156, + "mean_among_borrowers": 38670.33311547162, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.23072142160681058, + "mean_among_borrowers": 41126.05793612391, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1422795004455311, + "mean_among_borrowers": 52202.66543686702, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 21223403.433088087, + "mean_balance": 40446.915673129966, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.12533388559616318, + "5k_to_10k": 0.13601201669515878, + "10k_to_20k": 0.18997031373843593, + "20k_to_40k": 0.2351308957545589, + "40k_to_60k": 0.10824433293789903, + "60k_to_80k": 0.08668533742450127, + "80k_to_100k": 0.03163070469899026, + "100k_to_200k": 0.05604778033785523, + "200k_plus": 0.03094473281643728 + }, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257 + }, + "graduate_share_of_own_original_principal": 0.48269344604304165, + "ibr_new_borrower_share": 0.6147006160485154 + } + } + }, + "r3_household_income_drop_interest_dividends_and_ss_pension": { + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "scf_holdout": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "share": 0.1785134113597766, + "parent_plus_share": 0.12228737578012734, + "metrics_vs_scf": { + "share_with_debt_error_pp": -0.30361567622269603, + "mean_among_borrowers_error_pct": -7.119030826113226, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03158207267342461, + "head_age_band_share_mae_pp": 1.2596405967540016, + "race_share_mae_pp": 1.069077118976515, + "income_quintile_share_mae_pp": 0.6750791601254419, + "borrower_balance_band_tvd": 0.032948871945896414, + "borrower_age_band_tvd": 0.014803476410333236 + } + }, + "reason": "As r2 with the household income unit, to separate the two changes.", + "share": 0.16471154337166136, + "parent_plus_share": 0.21133541826981264, + "parent_plus_by_reference_age": { + "24_and_younger": 1.1813563767402388e-06, + "25_to_34": 0.002878811061981838, + "35_to_49": 0.03437594638384093, + "50_to_61": 0.47607636186118407, + "62_and_older": 0.7759316514071253 + }, + "by_head_age_band": { + "24_and_younger": 0.1992126713308297, + "25_to_34": 0.2819307615067069, + "35_to_49": 0.2452787877797859, + "50_to_61": 0.16522122503062336, + "62_and_older": 0.03883612510249803 + }, + "by_income_quintile": { + "1": 0.13542359408511098, + "2": 0.1539092744367252, + "3": 0.1590952051097958, + "4": 0.23485548375745907, + "5": 0.14025371248135685 + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.6838024750342213, + "mean_among_borrowers_error_pct": 4.439528929525149, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.04597736952235504, + "head_age_band_share_mae_pp": 2.6656550710525306, + "race_share_mae_pp": 1.196987234789608, + "income_quintile_share_mae_pp": 2.5224786119025513, + "borrower_balance_band_tvd": 0.03490274180911923, + "borrower_age_band_tvd": 0.05061741585504796 + }, + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16471154337166136, + "mean_among_borrowers": 42874.79958838091, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 865353807411.8156, + "parent_plus_share_of_total": 0.21133541826981264 + }, + "household_balance_bands": { + "under_5k": 0.11607469728946729, + "5k_to_10k": 0.13438073884084578, + "10k_to_20k": 0.18390441158440993, + "20k_to_40k": 0.2175213345370531, + "40k_to_60k": 0.11322429790687383, + "60k_to_80k": 0.09839303402018727, + "80k_to_100k": 0.040696765534088644, + "100k_to_200k": 0.06459753241505105, + "200k_plus": 0.031207187872023125 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.1992126713308297, + "mean_among_borrowers": 18109.488073489367, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.2819307615067069, + "mean_among_borrowers": 36270.682996325166, + "median_among_borrowers": 21300.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.2452787877797859, + "mean_among_borrowers": 44326.47466297097, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16522122503062336, + "mean_among_borrowers": 46739.99930437942, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.03883612510249803, + "mean_among_borrowers": 62130.09345503386, + "median_among_borrowers": 30000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15248141549588676, + "mean_among_borrowers": 43244.56985324743, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.2524562948063122, + "mean_among_borrowers": 43362.722253136395, + "median_among_borrowers": 27000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.1439203802335117, + "mean_among_borrowers": 38963.62666957163, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.17859030346275792, + "mean_among_borrowers": 45351.73574282507, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.13542359408511098, + "mean_among_borrowers": 42547.97877457715, + "median_among_borrowers": 19322.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1539092744367252, + "mean_among_borrowers": 39890.227134542816, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.1590952051097958, + "mean_among_borrowers": 35227.932190327105, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.23485548375745907, + "mean_among_borrowers": 43504.8501632244, + "median_among_borrowers": 27500.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.14025371248135685, + "mean_among_borrowers": 54083.17619953545, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 20998958.074320182, + "mean_balance": 41209.36878625729, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.12299613974060244, + "5k_to_10k": 0.13840505815198073, + "10k_to_20k": 0.18337471576757322, + "20k_to_40k": 0.2190436105053335, + "40k_to_60k": 0.11735011741410456, + "60k_to_80k": 0.09189840869829337, + "80k_to_100k": 0.03614263089100404, + "100k_to_200k": 0.061323665208594696, + "200k_plus": 0.029465653622513483 + }, + "age_bands": { + "24_and_younger": 0.04513484789969594, + "25_to_34": 0.2928507022678982, + "35_to_49": 0.3809737565496338, + "50_to_61": 0.20683966973224618, + "62_and_older": 0.07420102355052594 + }, + "graduate_share_of_own_original_principal": 0.5129961891047207, + "ibr_new_borrower_share": 0.6152171940032605 + } + } + } + } +} diff --git a/experiments/receipts/1073-scf-education-debt-fit.json b/experiments/receipts/1073-scf-education-debt-fit.json new file mode 100644 index 0000000000..4028104efa --- /dev/null +++ b/experiments/receipts/1073-scf-education-debt-fit.json @@ -0,0 +1,2151 @@ +{ + "schema_version": 1, + "issue": "PolicyEngine/microcosm#1073", + "generated_at": "2026-10-23", + "provenance": { + "commit": "60255dd9808ff1a0918e8c781b8818ce3376c32c", + "run": "20261023T152015Z-60255dd98" + }, + "scf_files": { + "rscfp2025.dta": { + "url": "https://www.federalreserve.gov/econres/files/scfp2025s.zip", + "zip_sha256": "32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + "member_sha256": "454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd" + }, + "p25i6.dta": { + "url": "https://www.federalreserve.gov/econres/files/scf2025s.zip", + "zip_sha256": "c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + "member_sha256": "38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad" + } + }, + "source_checks": { + "fed_alternate_table_13": { + "source": "Federal Reserve, 2025 SCF Chartbook tables, scf2025_tables_public_real_historical.xlsx, sheet 'Table 13 25 Alt %s & median', education loans", + "share_of_families_with_education_debt_pct": 19.92, + "median_among_holders": 25500.0 + }, + "loader_share_of_families_with_education_debt_pct": 19.916506070519297, + "loader_median_among_holders": 25500.0, + "families_per_implicate": 4360, + "balance_shares": { + "loans_1_6_of_all_education_debt": 0.9943433718962694, + "x7179_remaining_loans_of_all_education_debt": 0.0056566281037305766, + "federal_of_loans_1_6": 0.8742168884171948, + "non_federal_of_loans_1_6": 0.11976788840960186, + "federal_status_blank_of_loans_1_6": 0.006015223173203283 + }, + "federal_balance_share_by_whose_education_code": { + "1 respondent (reference person)": 0.5478808107398941, + "2 spouse or partner": 0.3138757801591027, + "3 child (incl. grandchild, other relative)": 0.13824340910100308, + "6 multiple people": 0.0, + "7 unrelated person": 0.0, + "-7 other": 0.0 + }, + "whose_education_codes_outside_the_rule": [], + "federal_education_debt_after_x7179_rule": 1005887392198.3959, + "remaining_loans_without_detail_dollars": 0.0, + "x8000_degree_check": [ + { + "loans": "1, 2, 3, 5", + "x8000": "not reversed", + "code": 1, + "discriminating": 180, + "consistent_with_coded_person": 175, + "discriminating_implicate_1": 36, + "consistent_implicate_1": 35 + }, + { + "loans": "1, 2, 3, 5", + "x8000": "not reversed", + "code": 2, + "discriminating": 208, + "consistent_with_coded_person": 203, + "discriminating_implicate_1": 40, + "consistent_implicate_1": 40 + }, + { + "loans": "1, 2, 3, 5", + "x8000": "reversed", + "code": 1, + "discriminating": 73, + "consistent_with_coded_person": 70, + "discriminating_implicate_1": 15, + "consistent_implicate_1": 14 + }, + { + "loans": "1, 2, 3, 5", + "x8000": "reversed", + "code": 2, + "discriminating": 470, + "consistent_with_coded_person": 470, + "discriminating_implicate_1": 92, + "consistent_implicate_1": 92 + }, + { + "loans": "4, 6", + "x8000": "not reversed", + "code": 1, + "discriminating": 20, + "consistent_with_coded_person": 20, + "discriminating_implicate_1": 4, + "consistent_implicate_1": 4 + }, + { + "loans": "4, 6", + "x8000": "not reversed", + "code": 2, + "discriminating": 5, + "consistent_with_coded_person": 5, + "discriminating_implicate_1": 1, + "consistent_implicate_1": 1 + }, + { + "loans": "4, 6", + "x8000": "reversed", + "code": 1, + "discriminating": 49, + "consistent_with_coded_person": 0, + "discriminating_implicate_1": 9, + "consistent_implicate_1": 0 + }, + { + "loans": "4, 6", + "x8000": "reversed", + "code": 2, + "discriminating": 5, + "consistent_with_coded_person": 0, + "discriminating_implicate_1": 1, + "consistent_implicate_1": 0 + } + ], + "child_education_readings": { + "before": { + "parent_plus_share_of_federal": 0.13750075307389345, + "child_own_share_of_federal": 0.0, + "parent_plus_dollars": 138310273934.81424, + "borrowers": 27234421.461229563, + "parent_plus_holders": 4381038.706092626, + "child_borrowers": 0.0 + }, + "x8000_only": { + "parent_plus_share_of_federal": 0.13750075307389345, + "child_own_share_of_federal": 0.0, + "parent_plus_dollars": 138310273934.81424, + "borrowers": 26923887.076506052, + "parent_plus_holders": 4381038.706092626, + "child_borrowers": 0.0 + }, + "build": { + "parent_plus_share_of_federal": 0.09068330207742185, + "child_own_share_of_federal": 0.04681745099647159, + "parent_plus_dollars": 91217190242.59724, + "borrowers": 27237098.83004764, + "parent_plus_holders": 2116082.8804507507, + "child_borrowers": 2347267.516216494 + }, + "within_limit_parent_plus": { + "parent_plus_share_of_federal": 0.11789995879009203, + "child_own_share_of_federal": 0.019600794283801418, + "parent_plus_dollars": 118594082087.66402, + "borrowers": 27076002.732565746, + "parent_plus_holders": 4144863.2081744918, + "child_borrowers": 388291.15397783055 + } + }, + "consolidated_share_of_federal_balances_dated_2014_or_later": 0.2928368593828854 + }, + "fsa": { + "files": { + "PortfolioSummary.xls": "11606d4b93ceea969fcae0f790f3a4e1c2999b62050559ce89bdc269810a62ee", + "PortfoliobyLoanType.xls": "e64e9d2945dfe703d42d383a7f65fbd4b044d134565ea77a0bd194f366becd33", + "Portfolio-by-Age.xls": "c2027d277bb88e31c5a8c0f8fc6d81422f45d60e0306f79e63a0c25c7286e2ef", + "Portfolio-by-Debt-Size.xls": "881bd8bbd257da5f5b906b0f0923cf246de6e3d28ee35e8be1cd23d841e88a4d" + }, + "total_dollars_fy2026_q3": 1724500000000.0, + "unduplicated_recipients_fy2026_q3": 42300000.0, + "parent_plus_dollars_fy2026_q3": 115000000000.0, + "parent_plus_recipients_fy2026_q3": 3500000.0, + "borrowers_by_age_fy2026_q2": { + "24_and_younger": 6200000.0, + "25_to_34": 14000000.0, + "35_to_49": 15000000.0, + "50_to_61": 6400000.0, + "62_and_older": 3200000.0, + "not_reported": 100000.0 + }, + "borrowers_by_debt_size_fy2026_q2": { + "under_5k": 6900000.0, + "5k_to_10k": 7200000.0, + "10k_to_20k": 9200000.0, + "20k_to_40k": 9500000.0, + "40k_to_60k": 4200000.0, + "60k_to_80k": 2600000.0, + "80k_to_100k": 1400000.0, + "100k_to_200k": 2600000.0, + "200k_plus": 1200000.0 + } + }, + "previous_receipt": { + "commit": "2a6e65a739b79a8232e1177610d11928fe92e26c", + "run": "20261022T153020Z-2a6e65a73", + "host_build": { + "parent_plus_share_of_total": 0.16758194869071996, + "federal_education_debt_total": 858421208954.9309, + "borrower_weight": 21223403.433088087, + "age_bands": { + "24_and_younger": 0.049988325664619956, + "25_to_34": 0.2917790224346921, + "35_to_49": 0.3780495021343625, + "50_to_61": 0.20778027670943267, + "62_and_older": 0.07240287305689257 + } + } + }, + "scf_2025_observed": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.18154956812200357, + "mean_among_borrowers": 41052.27209260245, + "median_among_borrowers": 25000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 1005887392198.3959, + "parent_plus_share_of_total": 0.09068330207742185 + }, + "household_balance_bands": { + "under_5k": 0.09639294713632403, + "5k_to_10k": 0.14003280730185305, + "10k_to_20k": 0.1763887243762958, + "20k_to_40k": 0.23167145676213327, + "40k_to_60k": 0.13228448002879964, + "60k_to_80k": 0.09356242722856564, + "80k_to_100k": 0.04781176224843039, + "100k_to_200k": 0.05847272768581226, + "200k_plus": 0.023382667231785843 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.22183897089190358, + "mean_among_borrowers": 23531.319738797185, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3615365625657711, + "mean_among_borrowers": 36363.72833498089, + "median_among_borrowers": 24900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.27186430754586627, + "mean_among_borrowers": 45297.2739464529, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.1666515695303606, + "mean_among_borrowers": 44901.11056030461, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.041870913769168856, + "mean_among_borrowers": 41544.318165395336, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.17240693757923897, + "mean_among_borrowers": 41150.07823437558, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.24343266231667623, + "mean_among_borrowers": 47375.69026752636, + "median_among_borrowers": 30000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.1409034533516125, + "mean_among_borrowers": 33382.63507234893, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.19450371139945485, + "mean_among_borrowers": 37451.352530441625, + "median_among_borrowers": 20000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.1144757325207629, + "mean_among_borrowers": 30747.429065421915, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.1705804998024808, + "mean_among_borrowers": 41222.638862750704, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17774016899539674, + "mean_among_borrowers": 38756.18543238133, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26595688145561025, + "mean_among_borrowers": 40379.14671664771, + "median_among_borrowers": 28000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.1790121945626286, + "mean_among_borrowers": 50757.93410007803, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 27237098.830047637, + "mean_balance": 36930.783211342365, + "median_balance": 21000.0, + "balance_bands": { + "under_5k": 0.11858724248854419, + "5k_to_10k": 0.15545017628617694, + "10k_to_20k": 0.18221344662589709, + "20k_to_40k": 0.22815323048913974, + "40k_to_60k": 0.12964994896026388, + "60k_to_80k": 0.07784527443403214, + "80k_to_100k": 0.03718546813313425, + "100k_to_200k": 0.04901089121852003, + "200k_plus": 0.021904321364291858 + }, + "age_bands": { + "24_and_younger": 0.10943214673515073, + "25_to_34": 0.3533895334655874, + "35_to_49": 0.37160638234348325, + "50_to_61": 0.11180436490051585, + "62_and_older": 0.05376757255526285 + }, + "graduate_share_of_own_original_principal": 0.47206971539270537, + "ibr_new_borrower_share": 0.5952213780001776 + } + }, + "scf_2025_observed_before": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.18154956812200357, + "mean_among_borrowers": 41052.27209260245, + "median_among_borrowers": 25000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 1005887392198.3959, + "parent_plus_share_of_total": 0.13750075307389345 + }, + "household_balance_bands": { + "under_5k": 0.09639294713632403, + "5k_to_10k": 0.14003280730185305, + "10k_to_20k": 0.1763887243762958, + "20k_to_40k": 0.23167145676213327, + "40k_to_60k": 0.13228448002879964, + "60k_to_80k": 0.09356242722856564, + "80k_to_100k": 0.04781176224843039, + "100k_to_200k": 0.05847272768581226, + "200k_plus": 0.023382667231785843 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.22183897089190358, + "mean_among_borrowers": 23531.319738797185, + "median_among_borrowers": 10000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3615365625657711, + "mean_among_borrowers": 36363.72833498089, + "median_among_borrowers": 24900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.27186430754586627, + "mean_among_borrowers": 45297.2739464529, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.1666515695303606, + "mean_among_borrowers": 44901.11056030461, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.041870913769168856, + "mean_among_borrowers": 41544.318165395336, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.17240693757923897, + "mean_among_borrowers": 41150.07823437558, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.24343266231667623, + "mean_among_borrowers": 47375.69026752636, + "median_among_borrowers": 30000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.1409034533516125, + "mean_among_borrowers": 33382.63507234893, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.19450371139945485, + "mean_among_borrowers": 37451.352530441625, + "median_among_borrowers": 20000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.1144757325207629, + "mean_among_borrowers": 30747.429065421915, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.1705804998024808, + "mean_among_borrowers": 41222.638862750704, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17774016899539674, + "mean_among_borrowers": 38756.18543238133, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26595688145561025, + "mean_among_borrowers": 40379.14671664771, + "median_among_borrowers": 28000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.1790121945626286, + "mean_among_borrowers": 50757.93410007803, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 27234421.461229563, + "mean_balance": 36934.413812695064, + "median_balance": 21000.0, + "balance_bands": { + "under_5k": 0.11892493269111482, + "5k_to_10k": 0.15531044452506518, + "10k_to_20k": 0.18601428472545406, + "20k_to_40k": 0.22258528435724315, + "40k_to_60k": 0.1281197371592323, + "60k_to_80k": 0.08084824950055876, + "80k_to_100k": 0.03718912377212063, + "100k_to_200k": 0.04910146852901287, + "200k_plus": 0.021906474740198353 + }, + "age_bands": { + "24_and_younger": 0.057262304197073094, + "25_to_34": 0.33134066182556904, + "35_to_49": 0.3807800839788549, + "50_to_61": 0.1615926964748789, + "62_and_older": 0.06902425352362417 + }, + "graduate_share_of_own_original_principal": 0.47738241507384116, + "ibr_new_borrower_share": 0.5915143468879298 + } + }, + "holdout": { + "folds": 5, + "seed": 0, + "n_estimators": 100, + "production_variant": "build_without_interest_and_ss_pension", + "variants": { + "shared_8_without_age": { + "predictors": [ + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17905210701848603, + "mean_among_borrowers": 40351.49510409681, + "median_among_borrowers": 24000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 975115394166.3627, + "parent_plus_share_of_total": 0.06206739011401023 + }, + "household_balance_bands": { + "under_5k": 0.10118107773912553, + "5k_to_10k": 0.13503432715253297, + "10k_to_20k": 0.18320533224863594, + "20k_to_40k": 0.21890176642533937, + "40k_to_60k": 0.13658109221694026, + "60k_to_80k": 0.1008575608439786, + "80k_to_100k": 0.045400028448036055, + "100k_to_200k": 0.05696009741342364, + "200k_plus": 0.021878717511987597 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.2856494213931131, + "mean_among_borrowers": 33090.297708091784, + "median_among_borrowers": 16000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.2634230533073922, + "mean_among_borrowers": 39094.6586979493, + "median_among_borrowers": 25000.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2511547364155009, + "mean_among_borrowers": 42005.58484858575, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.20137196508720936, + "mean_among_borrowers": 39931.548888414545, + "median_among_borrowers": 23000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.06509257053181323, + "mean_among_borrowers": 42567.96041221, + "median_among_borrowers": 28000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.16722261212813858, + "mean_among_borrowers": 40003.15075854964, + "median_among_borrowers": 24306.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.236098978018357, + "mean_among_borrowers": 45086.614893713966, + "median_among_borrowers": 27000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.15009122535417432, + "mean_among_borrowers": 34555.5308698397, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1987172729932262, + "mean_among_borrowers": 39734.161271935416, + "median_among_borrowers": 20200.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.1250378708596092, + "mean_among_borrowers": 32497.713789742633, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.16848970888296705, + "mean_among_borrowers": 40638.8255469805, + "median_among_borrowers": 23000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17317139260393358, + "mean_among_borrowers": 38343.90461862453, + "median_among_borrowers": 24000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26089759644033866, + "mean_among_borrowers": 43703.27046192123, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16768340855350833, + "mean_among_borrowers": 42778.330819886694, + "median_among_borrowers": 26000.0 + } + }, + "borrowers": { + "borrower_weight": 25895695.700705215, + "mean_balance": 37655.50095415307, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.11923381414468255, + "5k_to_10k": 0.13703997807327198, + "10k_to_20k": 0.18638889289819083, + "20k_to_40k": 0.21794950848449265, + "40k_to_60k": 0.1381837484113108, + "60k_to_80k": 0.09092222085828328, + "80k_to_100k": 0.03949213571886882, + "100k_to_200k": 0.0528776909925517, + "200k_plus": 0.017912010418347366 + }, + "age_bands": { + "24_and_younger": 0.12221762793944202, + "25_to_34": 0.2714652723519627, + "35_to_49": 0.3314329221489056, + "50_to_61": 0.17835091169774608, + "62_and_older": 0.09653326586194358 + }, + "graduate_share_of_own_original_principal": 0.4503453183190178, + "ibr_new_borrower_share": 0.5833971357123033 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.24974611035175343, + "mean_among_borrowers_error_pct": -1.7070358174692046, + "median_among_borrowers_error_pct": -4.0000000000000036, + "household_balance_band_tvd": 0.023196484278695206, + "head_age_band_share_mae_pp": 4.81151166418894, + "race_share_mae_pp": 0.6479835836438199, + "income_quintile_share_mae_pp": 0.6721955334843016, + "borrower_balance_band_tvd": 0.03260623116349648, + "borrower_age_band_tvd": 0.1220977213082023, + "household_rank_correlation": 0.15376716764930412, + "parent_plus_drawn_without_observed_parent_plus": 0.04628009643969982 + } + }, + "shared_8": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17622772670671366, + "mean_among_borrowers": 39369.48956513442, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 936377498601.3939, + "parent_plus_share_of_total": 0.07131444648553982 + }, + "household_balance_bands": { + "under_5k": 0.09351058185138737, + "5k_to_10k": 0.1630953404365325, + "10k_to_20k": 0.1722357902778136, + "20k_to_40k": 0.23003959537160001, + "40k_to_60k": 0.12001064378084277, + "60k_to_80k": 0.1082920780850408, + "80k_to_100k": 0.03372552360552188, + "100k_to_200k": 0.0612588533574213, + "200k_plus": 0.017831593233839888 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.2529895671878583, + "mean_among_borrowers": 24415.889909464808, + "median_among_borrowers": 11143.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3600484792377984, + "mean_among_borrowers": 35938.930360187805, + "median_among_borrowers": 23244.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2683793916575207, + "mean_among_borrowers": 41207.86453627056, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.14913648907893393, + "mean_among_borrowers": 41820.081246818496, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03583557010066891, + "mean_among_borrowers": 51625.44274182284, + "median_among_borrowers": 36000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.1630534163575822, + "mean_among_borrowers": 38771.91688875415, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.2289947160752199, + "mean_among_borrowers": 47907.93891717693, + "median_among_borrowers": 27000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.1566171160511706, + "mean_among_borrowers": 33168.61865028642, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1981094869447479, + "mean_among_borrowers": 35786.5815808536, + "median_among_borrowers": 20911.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.12126660088067846, + "mean_among_borrowers": 32702.7958886962, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.1559905239732057, + "mean_among_borrowers": 37712.13786074628, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17955131715072067, + "mean_among_borrowers": 38476.01343761174, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.26346013869266394, + "mean_among_borrowers": 40530.176936455646, + "median_among_borrowers": 23195.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16089043903093728, + "mean_among_borrowers": 45097.304974791696, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 25088578.38062776, + "mean_balance": 37322.86000407346, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.10677077633314812, + "5k_to_10k": 0.16632579551387022, + "10k_to_20k": 0.17454470788079757, + "20k_to_40k": 0.2286739571526593, + "40k_to_60k": 0.12048695309434337, + "60k_to_80k": 0.09765459818108711, + "80k_to_100k": 0.03345947236099234, + "100k_to_200k": 0.05493773460369269, + "200k_plus": 0.017146004879409302 + }, + "age_bands": { + "24_and_younger": 0.11420848773987989, + "25_to_34": 0.3588039745460583, + "35_to_49": 0.3516343204975256, + "50_to_61": 0.11346958485303148, + "62_and_older": 0.061883632363504804 + }, + "graduate_share_of_own_original_principal": 0.49107247959697303, + "ibr_new_borrower_share": 0.570162419212365 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.5321841415289907, + "mean_among_borrowers_error_pct": -4.099121538686434, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.040578309662763545, + "head_age_band_share_mae_pp": 1.1934803926439914, + "race_share_mae_pp": 1.077772642699106, + "income_quintile_share_mae_pp": 0.8762098127830446, + "borrower_balance_band_tvd": 0.03713251302344053, + "borrower_age_band_tvd": 0.019972061845957664, + "household_rank_correlation": 0.17266749563083275, + "parent_plus_drawn_without_observed_parent_plus": 0.02526658514008899 + } + }, + "shared_8_plus_household_income": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17281627661600854, + "mean_among_borrowers": 39254.97869162569, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 915580080877.1456, + "parent_plus_share_of_total": 0.059940399569770764 + }, + "household_balance_bands": { + "under_5k": 0.08974701999643789, + "5k_to_10k": 0.14806900868476966, + "10k_to_20k": 0.1780192726055367, + "20k_to_40k": 0.2452616168183236, + "40k_to_60k": 0.1205246550102769, + "60k_to_80k": 0.10657649709866371, + "80k_to_100k": 0.039753518945480334, + "100k_to_200k": 0.05175325944273187, + "200k_plus": 0.020295151397779383 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.24417812288361246, + "mean_among_borrowers": 29778.191701476153, + "median_among_borrowers": 11000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.3474148475378485, + "mean_among_borrowers": 36809.50492816768, + "median_among_borrowers": 24600.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2717612000881406, + "mean_among_borrowers": 39305.59411440302, + "median_among_borrowers": 22000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.14495396516593925, + "mean_among_borrowers": 42162.711454936776, + "median_among_borrowers": 28000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03258898091103795, + "mean_among_borrowers": 51769.49362308479, + "median_among_borrowers": 28000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.15882220643031406, + "mean_among_borrowers": 38309.93027102956, + "median_among_borrowers": 24000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.224885334931214, + "mean_among_borrowers": 44951.2117460371, + "median_among_borrowers": 24600.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.16095386588360475, + "mean_among_borrowers": 34893.011406958016, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1923427751654244, + "mean_among_borrowers": 39120.753536492375, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.10260826272789049, + "mean_among_borrowers": 33878.87020012928, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.17100662350830068, + "mean_among_borrowers": 39154.50811835669, + "median_among_borrowers": 22000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.1785510426393907, + "mean_among_borrowers": 36629.24348561517, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.25540364615718963, + "mean_among_borrowers": 37772.04606554361, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.1565282419252913, + "mean_among_borrowers": 48302.23313145786, + "median_among_borrowers": 26000.0 + } + }, + "borrowers": { + "borrower_weight": 24592745.901716128, + "mean_balance": 37229.68083906623, + "median_balance": 22000.0, + "balance_bands": { + "under_5k": 0.10360174630285794, + "5k_to_10k": 0.1504219956091371, + "10k_to_20k": 0.18311213927672274, + "20k_to_40k": 0.23971269075178334, + "40k_to_60k": 0.12279030310946719, + "60k_to_80k": 0.09669836723100285, + "80k_to_100k": 0.03608342021771613, + "100k_to_200k": 0.05000127236763974, + "200k_plus": 0.017578065133672955 + }, + "age_bands": { + "24_and_younger": 0.10880795012149283, + "25_to_34": 0.35906309742386944, + "35_to_49": 0.36721906946956695, + "50_to_61": 0.10788831666800004, + "62_and_older": 0.057021566317070796 + }, + "graduate_share_of_own_original_principal": 0.4592646793399742, + "ibr_new_borrower_share": 0.5760474012257303 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.8733291505995028, + "mean_among_borrowers_error_pct": -4.378060724440691, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.03627097953844585, + "head_age_band_share_mae_pp": 1.350870233998188, + "race_share_mae_pp": 1.3585851825102468, + "income_quintile_share_mae_pp": 0.9228331015688831, + "borrower_balance_band_tvd": 0.032301626859559736, + "borrower_age_band_tvd": 0.008927557720090004, + "household_rank_correlation": 0.17512968016954317, + "parent_plus_drawn_without_observed_parent_plus": 0.02654694891508374 + } + }, + "plus_household_income_and_education": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income", + "educ" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.1668063934073779, + "mean_among_borrowers": 38760.6787211115, + "median_among_borrowers": 23796.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 872611659274.6003, + "parent_plus_share_of_total": 0.07414637298387526 + }, + "household_balance_bands": { + "under_5k": 0.08808431892789516, + "5k_to_10k": 0.13717658256161608, + "10k_to_20k": 0.18882635422463928, + "20k_to_40k": 0.2453633644579256, + "40k_to_60k": 0.12322636723840236, + "60k_to_80k": 0.10776956790540955, + "80k_to_100k": 0.03911980624752822, + "100k_to_200k": 0.05591384389558031, + "200k_plus": 0.014519794541003451 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.22536433723499413, + "mean_among_borrowers": 33402.49807525617, + "median_among_borrowers": 16000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.35882688319565326, + "mean_among_borrowers": 36572.66097268626, + "median_among_borrowers": 24900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.25221217813313995, + "mean_among_borrowers": 38787.71300020815, + "median_among_borrowers": 22900.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.13669823926255414, + "mean_among_borrowers": 40230.335480571295, + "median_among_borrowers": 25950.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03165003376804131, + "mean_among_borrowers": 50489.727264864836, + "median_among_borrowers": 37000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.156024839562724, + "mean_among_borrowers": 38481.303452352964, + "median_among_borrowers": 23666.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.19389802688054777, + "mean_among_borrowers": 40739.89648452815, + "median_among_borrowers": 25166.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.15371847235012892, + "mean_among_borrowers": 32038.772885804257, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.1984583260312187, + "mean_among_borrowers": 42274.537980522124, + "median_among_borrowers": 25000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.09539828169554497, + "mean_among_borrowers": 43336.23834084036, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.16600786857086094, + "mean_among_borrowers": 39624.46738359425, + "median_among_borrowers": 21939.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.1671000542140161, + "mean_among_borrowers": 34192.31543281778, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.25267861183865514, + "mean_among_borrowers": 34599.69654081462, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.15286484625519245, + "mean_among_borrowers": 46837.093972152856, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 23714170.340499215, + "mean_balance": 36797.056221880484, + "median_balance": 23000.0, + "balance_bands": { + "under_5k": 0.09699471892619613, + "5k_to_10k": 0.1394638892424441, + "10k_to_20k": 0.18911050663109966, + "20k_to_40k": 0.2551113824446505, + "40k_to_60k": 0.12484286667522601, + "60k_to_80k": 0.09515433282795518, + "80k_to_100k": 0.03559212688986377, + "100k_to_200k": 0.049877581326660494, + "200k_plus": 0.013852595035904155 + }, + "age_bands": { + "24_and_younger": 0.10242792997260068, + "25_to_34": 0.3760811152011804, + "35_to_49": 0.3477655420020849, + "50_to_61": 0.11573428915655427, + "62_and_older": 0.05799112366757967 + }, + "graduate_share_of_own_original_principal": 0.421014411591174, + "ibr_new_borrower_share": 0.564984415474899 + } + }, + "metrics": { + "share_with_debt_error_pp": -1.4743174714625662, + "mean_among_borrowers_error_pct": -5.582135298922686, + "median_among_borrowers_error_pct": -4.815999999999998, + "household_balance_band_tvd": 0.04033667822097967, + "head_age_band_share_mae_pp": 1.3212277078973742, + "race_share_mae_pp": 2.067159177073093, + "income_quintile_share_mae_pp": 1.4743162952521935, + "borrower_balance_band_tvd": 0.052030960462776894, + "borrower_age_band_tvd": 0.030845057103948335, + "household_rank_correlation": 0.20793128913160808, + "parent_plus_drawn_without_observed_parent_plus": 0.01636273473995983 + } + }, + "build_without_interest_and_ss_pension": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.1785134113597766, + "mean_among_borrowers": 38156.40076634909, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 919295632615.8619, + "parent_plus_share_of_total": 0.06816282381081624 + }, + "household_balance_bands": { + "under_5k": 0.09126324438556496, + "5k_to_10k": 0.15516041383614745, + "10k_to_20k": 0.1786625820914109, + "20k_to_40k": 0.24094330640977704, + "40k_to_60k": 0.12235248869076151, + "60k_to_80k": 0.09731073610543892, + "80k_to_100k": 0.04628423532619501, + "100k_to_200k": 0.04868017050483829, + "200k_plus": 0.019342822649865955 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.24959547861591452, + "mean_among_borrowers": 30234.050260438173, + "median_among_borrowers": 14000.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.36900446914360446, + "mean_among_borrowers": 35924.35022877094, + "median_among_borrowers": 23000.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.2743786349921627, + "mean_among_borrowers": 38619.55355856089, + "median_among_borrowers": 22000.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.14829262594950599, + "mean_among_borrowers": 40031.502537207765, + "median_among_borrowers": 27000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03498656926046411, + "mean_among_borrowers": 48239.97130082438, + "median_among_borrowers": 27000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.1672189930591373, + "mean_among_borrowers": 36915.04739897259, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.2324868349178142, + "mean_among_borrowers": 44721.075062431184, + "median_among_borrowers": 25000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.16156695468894305, + "mean_among_borrowers": 35711.49600365523, + "median_among_borrowers": 20000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.18853789989668854, + "mean_among_borrowers": 36645.789525487984, + "median_among_borrowers": 21000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.10379714978765006, + "mean_among_borrowers": 34621.48654269449, + "median_among_borrowers": 19200.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.17803919929996376, + "mean_among_borrowers": 37503.22075890366, + "median_among_borrowers": 21000.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17956752806930856, + "mean_among_borrowers": 35880.44316170545, + "median_among_borrowers": 21000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.2632482800176951, + "mean_among_borrowers": 38136.3817053988, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16793147929877925, + "mean_among_borrowers": 43498.44059935984, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 25810010.594911456, + "mean_balance": 35617.79369424609, + "median_balance": 21000.0, + "balance_bands": { + "under_5k": 0.10504821975483239, + "5k_to_10k": 0.1612084637239687, + "10k_to_20k": 0.1835278083390002, + "20k_to_40k": 0.2403341878392152, + "40k_to_60k": 0.1258185662605132, + "60k_to_80k": 0.08586155919483465, + "80k_to_100k": 0.03690144692037484, + "100k_to_200k": 0.04425676441601652, + "200k_plus": 0.01704298355124438 + }, + "age_bands": { + "24_and_younger": 0.10884214784928073, + "25_to_34": 0.3590396433749492, + "35_to_49": 0.36321220703040163, + "50_to_61": 0.10958470645086948, + "62_and_older": 0.05932129529449908 + }, + "graduate_share_of_own_original_principal": 0.4472065188684173, + "ibr_new_borrower_share": 0.5842486917589438 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.30361567622269603, + "mean_among_borrowers_error_pct": -7.054107309142554, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.030421622773926488, + "head_age_band_share_mae_pp": 1.2596405967540016, + "race_share_mae_pp": 1.069077118976515, + "income_quintile_share_mae_pp": 0.6750791601254419, + "borrower_balance_band_tvd": 0.027269891261772868, + "borrower_age_band_tvd": 0.011203832648598016, + "household_rank_correlation": 0.1678412360138977, + "parent_plus_drawn_without_observed_parent_plus": 0.030096797445698646 + } + }, + "build_plus_education": { + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income", + "educ" + ], + "fit": { + "households": { + "household_weight": 134963690.69015205, + "share_with_federal_education_debt": 0.17234700775018835, + "mean_among_borrowers": 38146.7986991102, + "median_among_borrowers": 23000.0, + "total_household_weight": 134963690.69015205, + "federal_education_debt_total": 887316977419.0054, + "parent_plus_share_of_total": 0.0610214959298925 + }, + "household_balance_bands": { + "under_5k": 0.094500931393709, + "5k_to_10k": 0.1398997895474437, + "10k_to_20k": 0.19368288257466282, + "20k_to_40k": 0.23884340819629526, + "40k_to_60k": 0.13523397562363007, + "60k_to_80k": 0.09436055969371929, + "80k_to_100k": 0.03446366840523407, + "100k_to_200k": 0.052172845962700054, + "200k_plus": 0.01684193860260577 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 5853521.0, + "share_with_federal_education_debt": 0.24556070748817072, + "mean_among_borrowers": 29711.57837171095, + "median_among_borrowers": 13345.0 + }, + "25_to_34": { + "household_weight": 20594001.0, + "share_with_federal_education_debt": 0.37451305639803345, + "mean_among_borrowers": 36288.44136390066, + "median_among_borrowers": 23900.0 + }, + "35_to_49": { + "household_weight": 34447794.528511114, + "share_with_federal_education_debt": 0.25660807838739297, + "mean_among_borrowers": 38705.03441536813, + "median_among_borrowers": 21600.0 + }, + "50_to_61": { + "household_weight": 26383416.705058806, + "share_with_federal_education_debt": 0.13992324697554848, + "mean_among_borrowers": 38605.840676442705, + "median_among_borrowers": 25000.0 + }, + "62_and_older": { + "household_weight": 47684957.45658214, + "share_with_federal_education_debt": 0.03311810732872758, + "mean_among_borrowers": 50702.58315301913, + "median_among_borrowers": 30000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 81838505.71507175, + "share_with_federal_education_debt": 0.16168801258725676, + "mean_among_borrowers": 37237.88141376128, + "median_among_borrowers": 22900.0 + }, + "black_non_hispanic": { + "household_weight": 18706038.359267425, + "share_with_federal_education_debt": 0.20914133036844892, + "mean_among_borrowers": 44561.07772792855, + "median_among_borrowers": 26000.0 + }, + "hispanic": { + "household_weight": 15955872.224651583, + "share_with_federal_education_debt": 0.16318201246763214, + "mean_among_borrowers": 31764.490083228888, + "median_among_borrowers": 21000.0 + }, + "other_or_multiple": { + "household_weight": 18463274.3911613, + "share_with_federal_education_debt": 0.19023526055566922, + "mean_among_borrowers": 39157.735848089564, + "median_among_borrowers": 20000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 26990442.923403554, + "share_with_federal_education_debt": 0.10129901303895304, + "mean_among_borrowers": 42632.530633246395, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 26994482.864333212, + "share_with_federal_education_debt": 0.17021674724220964, + "mean_among_borrowers": 39357.02146920614, + "median_among_borrowers": 20100.0 + }, + "3": { + "household_weight": 26997020.193277217, + "share_with_federal_education_debt": 0.17533106736677012, + "mean_among_borrowers": 31912.544492445948, + "median_among_borrowers": 20000.0 + }, + "4": { + "household_weight": 26985790.839470405, + "share_with_federal_education_debt": 0.2535480923849969, + "mean_among_borrowers": 35764.93532409649, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 26995953.869667668, + "share_with_federal_education_debt": 0.16135595041628936, + "mean_among_borrowers": 44570.452510837786, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 24629278.168691896, + "mean_balance": 36026.91769290015, + "median_balance": 20713.0, + "balance_bands": { + "under_5k": 0.11003236440751846, + "5k_to_10k": 0.1438146762421388, + "10k_to_20k": 0.1980177667743084, + "20k_to_40k": 0.2357539379328877, + "40k_to_60k": 0.13243383736799277, + "60k_to_80k": 0.08655648338436131, + "80k_to_100k": 0.030271579819877544, + "100k_to_200k": 0.048972446522624176, + "200k_plus": 0.014146907548290621 + }, + "age_bands": { + "24_and_younger": 0.10435819675485504, + "25_to_34": 0.384565596966415, + "35_to_49": 0.34123822245830965, + "50_to_61": 0.10931470172073098, + "62_and_older": 0.060523282099689185 + }, + "graduate_share_of_own_original_principal": 0.4374325644888308, + "ibr_new_borrower_share": 0.5785380464302201 + } + }, + "metrics": { + "share_with_debt_error_pp": -0.9202560371815216, + "mean_among_borrowers_error_pct": -7.077497164927471, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.028213737692513032, + "head_age_band_share_mae_pp": 1.7487117716451235, + "race_share_mae_pp": 1.7889316725003699, + "income_quintile_share_mae_pp": 0.920292137753204, + "borrower_balance_band_tvd": 0.034900124950217506, + "borrower_age_band_tvd": 0.037931773045254066, + "household_rank_correlation": 0.20717780916045597, + "parent_plus_drawn_without_observed_parent_plus": 0.02425600966419175 + } + } + } + }, + "host_settings": { + "first_receipt": { + "label": "First receipt", + "definition": { + "reference_person": "oldest_adult", + "partnered": "married_spouse", + "children": "of_reference", + "income_unit": "household" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14412999386354491, + "mean_among_borrowers": 44483.09961323457, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 785628046119.6727, + "parent_plus_share_of_total": 0.13018594263257258 + }, + "household_balance_bands": { + "under_5k": 0.09597901426395242, + "5k_to_10k": 0.1620676904114682, + "10k_to_20k": 0.15178493618645325, + "20k_to_40k": 0.22796648343692355, + "40k_to_60k": 0.09915026443778263, + "60k_to_80k": 0.10849192930212849, + "80k_to_100k": 0.056043492646084624, + "100k_to_200k": 0.06743544596300373, + "200k_plus": 0.031080743352203163 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 2628112.658323748, + "share_with_federal_education_debt": 0.2267174320421409, + "mean_among_borrowers": 18670.63953159211, + "median_among_borrowers": 6000.0 + }, + "25_to_34": { + "household_weight": 17516284.279591538, + "share_with_federal_education_debt": 0.2669429791293293, + "mean_among_borrowers": 36447.86184035785, + "median_among_borrowers": 20200.0 + }, + "35_to_49": { + "household_weight": 30963807.6460925, + "share_with_federal_education_debt": 0.22339688829964577, + "mean_among_borrowers": 44315.447461546944, + "median_among_borrowers": 25670.0 + }, + "50_to_61": { + "household_weight": 27085763.89545781, + "share_with_federal_education_debt": 0.14218578094851128, + "mean_among_borrowers": 49552.354866035355, + "median_among_borrowers": 29000.0 + }, + "62_and_older": { + "household_weight": 44343123.406217694, + "share_with_federal_education_debt": 0.03655929815086838, + "mean_among_borrowers": 65818.86253190483, + "median_among_borrowers": 38000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 80232895.31111473, + "share_with_federal_education_debt": 0.13333034770117472, + "mean_among_borrowers": 42577.59434933696, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 13933034.37446032, + "share_with_federal_education_debt": 0.22824902667788102, + "mean_among_borrowers": 47993.699863516216, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18643938.505382426, + "share_with_federal_education_debt": 0.12291729609744638, + "mean_among_borrowers": 46285.94561484015, + "median_among_borrowers": 24764.0 + }, + "other_or_multiple": { + "household_weight": 9727223.69472581, + "share_with_federal_education_debt": 0.15337638725400463, + "mean_among_borrowers": 47893.571558628166, + "median_among_borrowers": 24000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24506450.631004497, + "share_with_federal_education_debt": 0.12721563759800417, + "mean_among_borrowers": 38171.39283093021, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24508606.10032361, + "share_with_federal_education_debt": 0.1358771691149805, + "mean_among_borrowers": 45461.58729969715, + "median_among_borrowers": 26000.0 + }, + "3": { + "household_weight": 24502185.328004334, + "share_with_federal_education_debt": 0.13382646407799811, + "mean_among_borrowers": 40922.60303950533, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24513536.74023802, + "share_with_federal_education_debt": 0.19993030155269514, + "mean_among_borrowers": 45004.488576349744, + "median_among_borrowers": 25000.0 + }, + "5": { + "household_weight": 24506313.086112835, + "share_with_federal_education_debt": 0.12378308052696536, + "mean_among_borrowers": 52902.03665110157, + "median_among_borrowers": 28000.0 + } + }, + "borrowers": { + "borrower_weight": 18782588.57019364, + "mean_balance": 41827.46394000224, + "median_balance": 24700.0, + "balance_bands": { + "under_5k": 0.11010465011675885, + "5k_to_10k": 0.15766080070087685, + "10k_to_20k": 0.15145555688552018, + "20k_to_40k": 0.23876632487497312, + "40k_to_60k": 0.10328337810598322, + "60k_to_80k": 0.09709706504826032, + "80k_to_100k": 0.04973601963818461, + "100k_to_200k": 0.06276278750614343, + "200k_plus": 0.029133417123299275 + }, + "age_bands": { + "24_and_younger": 0.11708185382308234, + "25_to_34": 0.30330802369313054, + "35_to_49": 0.3722441005630567, + "50_to_61": 0.13909573807362918, + "62_and_older": 0.06827028384710118 + }, + "graduate_share_of_own_original_principal": 0.5079846787675676, + "ibr_new_borrower_share": 0.5790424319437123 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7419574258458654, + "mean_among_borrowers_error_pct": 8.357217142313411, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.061856909978440955, + "head_age_band_share_mae_pp": 3.5543373606609885, + "race_share_mae_pp": 2.8343426729118955, + "income_quintile_share_mae_pp": 4.2522526924143715, + "borrower_balance_band_tvd": 0.06560705296644279, + "borrower_age_band_tvd": 0.050081509772456775 + } + }, + "scf_codings_all_candidates": { + "label": "SCF codings, all predictors", + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "interest_dividend_income", + "social_security_pension_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.14453729401731658, + "mean_among_borrowers": 42173.85469504409, + "median_among_borrowers": 25000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 746948718213.9073, + "parent_plus_share_of_total": 0.10695080788545275 + }, + "household_balance_bands": { + "under_5k": 0.10775015906187184, + "5k_to_10k": 0.1528994221663368, + "10k_to_20k": 0.15878382717932804, + "20k_to_40k": 0.23707174823778418, + "40k_to_60k": 0.11254827709408233, + "60k_to_80k": 0.09506114425698398, + "80k_to_100k": 0.044996423198641146, + "100k_to_200k": 0.05939892961997767, + "200k_plus": 0.03149006918499395 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.2109485663269708, + "mean_among_borrowers": 22164.502413760794, + "median_among_borrowers": 14000.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.2709152074679265, + "mean_among_borrowers": 36647.47528338518, + "median_among_borrowers": 21600.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.20910334189997756, + "mean_among_borrowers": 42586.38683331958, + "median_among_borrowers": 25000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.1331061797690564, + "mean_among_borrowers": 49336.50631455528, + "median_among_borrowers": 30000.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.03120975131828736, + "mean_among_borrowers": 58478.07907213402, + "median_among_borrowers": 26000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.13312784049029713, + "mean_among_borrowers": 40718.755628997955, + "median_among_borrowers": 25000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.22788477777157826, + "mean_among_borrowers": 47824.23980122162, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.11755065774394279, + "mean_among_borrowers": 39599.36261332915, + "median_among_borrowers": 23000.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.1698883489488461, + "mean_among_borrowers": 44027.69290376679, + "median_among_borrowers": 22000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.13612388616760965, + "mean_among_borrowers": 38386.48367756073, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.1459793402381904, + "mean_among_borrowers": 42114.80663102074, + "median_among_borrowers": 23000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.13033686972701333, + "mean_among_borrowers": 39083.07983863153, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.1919836615881704, + "mean_among_borrowers": 42068.527873712716, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.11826299055685831, + "mean_among_borrowers": 50185.22911179778, + "median_among_borrowers": 26814.0 + } + }, + "borrowers": { + "borrower_weight": 18644433.43176055, + "mean_balance": 40062.82738211235, + "median_balance": 24000.0, + "balance_bands": { + "under_5k": 0.11847637897193393, + "5k_to_10k": 0.15215298880238085, + "10k_to_20k": 0.15903005076767437, + "20k_to_40k": 0.24355676444848492, + "40k_to_60k": 0.11418906569388854, + "60k_to_80k": 0.09072876578657417, + "80k_to_100k": 0.035563245230000456, + "100k_to_200k": 0.05657395317657679, + "200k_plus": 0.02972878712248597 + }, + "age_bands": { + "24_and_younger": 0.11887683513812723, + "25_to_34": 0.31829496102827626, + "35_to_49": 0.3727342252269422, + "50_to_61": 0.13256885684078035, + "62_and_older": 0.057525121765873946 + }, + "graduate_share_of_own_original_principal": 0.4878619324532848, + "ibr_new_borrower_share": 0.5971032517443657 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -3.7012274104686993, + "mean_among_borrowers_error_pct": 2.7320841095266557, + "median_among_borrowers_error_pct": 0.0, + "household_balance_band_tvd": 0.040156439181474324, + "head_age_band_share_mae_pp": 4.169585550417035, + "race_share_mae_pp": 2.569878492307957, + "income_quintile_share_mae_pp": 4.567500727054615, + "borrower_balance_band_tvd": 0.04367455302813815, + "borrower_age_band_tvd": 0.035094572437311095 + } + }, + "build": { + "label": "Build", + "definition": { + "reference_person": "householder_couple", + "partnered": "spouse_or_partner", + "children": "of_reference_or_partner", + "income_unit": "reference_couple_and_children" + }, + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16413671000206656, + "mean_among_borrowers": 42680.26879514089, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 858421208954.9309, + "parent_plus_share_of_total": 0.10691085971286082 + }, + "household_balance_bands": { + "under_5k": 0.1179407532730876, + "5k_to_10k": 0.1324473758996882, + "10k_to_20k": 0.1827494270561508, + "20k_to_40k": 0.23417620772637385, + "40k_to_60k": 0.10433535383061796, + "60k_to_80k": 0.0976346772652531, + "80k_to_100k": 0.0379014108379451, + "100k_to_200k": 0.05781213394341271, + "200k_plus": 0.03500266016747067 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.21797195752357604, + "mean_among_borrowers": 20475.641978208863, + "median_among_borrowers": 12662.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28107974022687005, + "mean_among_borrowers": 34573.95711939027, + "median_among_borrowers": 20100.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.24429588401566177, + "mean_among_borrowers": 44478.05328490035, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16352578267462928, + "mean_among_borrowers": 48394.24067058906, + "median_among_borrowers": 28742.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.037347739302272136, + "mean_among_borrowers": 61031.6733341336, + "median_among_borrowers": 34000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15291016385941802, + "mean_among_borrowers": 42266.19101485241, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.24861685783459786, + "mean_among_borrowers": 45500.64480289935, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.13835157436352993, + "mean_among_borrowers": 37031.934121143146, + "median_among_borrowers": 19200.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.18407544859054642, + "mean_among_borrowers": 48161.13603160578, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14257834053880739, + "mean_among_borrowers": 43328.84042017116, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.15791199105287693, + "mean_among_borrowers": 39522.35712436794, + "median_among_borrowers": 20000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14719109419650156, + "mean_among_borrowers": 38670.33311547162, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.23072142160681058, + "mean_among_borrowers": 41126.05793612391, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1422795004455311, + "mean_among_borrowers": 52202.66543686702, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 21177503.938145176, + "mean_balance": 40534.57912047568, + "median_balance": 22224.0, + "balance_bands": { + "under_5k": 0.12755450607386423, + "5k_to_10k": 0.13657317586234458, + "10k_to_20k": 0.18059640738658841, + "20k_to_40k": 0.2390426854563625, + "40k_to_60k": 0.10720974798612737, + "60k_to_80k": 0.09004176213429058, + "80k_to_100k": 0.034401838885435285, + "100k_to_200k": 0.051526234839151065, + "200k_plus": 0.03305364137583606 + }, + "age_bands": { + "24_and_younger": 0.12191368948550384, + "25_to_34": 0.3111513455503956, + "35_to_49": 0.37171601251334024, + "50_to_61": 0.13719732675785898, + "62_and_older": 0.05802162569290144 + }, + "graduate_share_of_own_original_principal": 0.4995084468534534, + "ibr_new_borrower_share": 0.6132625665271612 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.7412858119937007, + "mean_among_borrowers_error_pct": 3.965667719599386, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.046105502753231414, + "head_age_band_share_mae_pp": 2.3908244112012222, + "race_share_mae_pp": 0.9415277758683394, + "income_quintile_share_mae_pp": 2.865766910648814, + "borrower_balance_band_tvd": 0.0457178698849765, + "borrower_age_band_tvd": 0.04223818791519181 + } + } + }, + "host": { + "release": "populace-us-2024-spm-20260915", + "path": "populace_us_2024.h5", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "is_the_published_default": true, + "households": 57240, + "people": 166321, + "seed": 0, + "gate_passed": true, + "gate_failures": [], + "household_income_columns_present": [ + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "farm_operations_income", + "farm_rent_income", + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "rental_income", + "estate_income", + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents", + "taxable_private_pension_income", + "tax_exempt_private_pension_income", + "taxable_ira_distributions", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_sep_distributions", + "unemployment_compensation", + "alimony_income", + "child_support_received", + "veterans_benefits", + "workers_compensation", + "disability_benefits" + ], + "fit": { + "households": { + "household_weight": 122537091.8856833, + "share_with_federal_education_debt": 0.16413671000206656, + "mean_among_borrowers": 42680.26879514089, + "median_among_borrowers": 23000.0, + "total_household_weight": 122537091.8856833, + "federal_education_debt_total": 858421208954.9309, + "parent_plus_share_of_total": 0.10691085971286082 + }, + "household_balance_bands": { + "under_5k": 0.1179407532730876, + "5k_to_10k": 0.1324473758996882, + "10k_to_20k": 0.1827494270561508, + "20k_to_40k": 0.23417620772637385, + "40k_to_60k": 0.10433535383061796, + "60k_to_80k": 0.0976346772652531, + "80k_to_100k": 0.0379014108379451, + "100k_to_200k": 0.05781213394341271, + "200k_plus": 0.03500266016747067 + }, + "by_head_age_band": { + "24_and_younger": { + "household_weight": 4363759.075237429, + "share_with_federal_education_debt": 0.21797195752357604, + "mean_among_borrowers": 20475.641978208863, + "median_among_borrowers": 12662.0 + }, + "25_to_34": { + "household_weight": 19671660.332724698, + "share_with_federal_education_debt": 0.28107974022687005, + "mean_among_borrowers": 34573.95711939027, + "median_among_borrowers": 20100.0 + }, + "35_to_49": { + "household_weight": 32403635.225158654, + "share_with_federal_education_debt": 0.24429588401566177, + "mean_among_borrowers": 44478.05328490035, + "median_among_borrowers": 23000.0 + }, + "50_to_61": { + "household_weight": 25738755.87416564, + "share_with_federal_education_debt": 0.16352578267462928, + "mean_among_borrowers": 48394.24067058906, + "median_among_borrowers": 28742.0 + }, + "62_and_older": { + "household_weight": 40359281.378396876, + "share_with_federal_education_debt": 0.037347739302272136, + "mean_among_borrowers": 61031.6733341336, + "median_among_borrowers": 34000.0 + } + }, + "by_race": { + "white_non_hispanic": { + "household_weight": 79908332.17160814, + "share_with_federal_education_debt": 0.15291016385941802, + "mean_among_borrowers": 42266.19101485241, + "median_among_borrowers": 23000.0 + }, + "black_non_hispanic": { + "household_weight": 14037284.65604804, + "share_with_federal_education_debt": 0.24861685783459786, + "mean_among_borrowers": 45500.64480289935, + "median_among_borrowers": 28000.0 + }, + "hispanic": { + "household_weight": 18783518.810774647, + "share_with_federal_education_debt": 0.13835157436352993, + "mean_among_borrowers": 37031.934121143146, + "median_among_borrowers": 19200.0 + }, + "other_or_multiple": { + "household_weight": 9807956.247252459, + "share_with_federal_education_debt": 0.18407544859054642, + "mean_among_borrowers": 48161.13603160578, + "median_among_borrowers": 18000.0 + } + }, + "by_income_quintile": { + "1": { + "household_weight": 24515062.61657284, + "share_with_federal_education_debt": 0.14257834053880739, + "mean_among_borrowers": 43328.84042017116, + "median_among_borrowers": 20000.0 + }, + "2": { + "household_weight": 24494456.917873796, + "share_with_federal_education_debt": 0.15791199105287693, + "mean_among_borrowers": 39522.35712436794, + "median_among_borrowers": 20000.0 + }, + "3": { + "household_weight": 24510469.334416572, + "share_with_federal_education_debt": 0.14719109419650156, + "mean_among_borrowers": 38670.33311547162, + "median_among_borrowers": 23000.0 + }, + "4": { + "household_weight": 24509847.230985664, + "share_with_federal_education_debt": 0.23072142160681058, + "mean_among_borrowers": 41126.05793612391, + "median_among_borrowers": 26000.0 + }, + "5": { + "household_weight": 24507255.785834424, + "share_with_federal_education_debt": 0.1422795004455311, + "mean_among_borrowers": 52202.66543686702, + "median_among_borrowers": 23000.0 + } + }, + "borrowers": { + "borrower_weight": 21177503.938145176, + "mean_balance": 40534.57912047568, + "median_balance": 22224.0, + "balance_bands": { + "under_5k": 0.12755450607386423, + "5k_to_10k": 0.13657317586234458, + "10k_to_20k": 0.18059640738658841, + "20k_to_40k": 0.2390426854563625, + "40k_to_60k": 0.10720974798612737, + "60k_to_80k": 0.09004176213429058, + "80k_to_100k": 0.034401838885435285, + "100k_to_200k": 0.051526234839151065, + "200k_plus": 0.03305364137583606 + }, + "age_bands": { + "24_and_younger": 0.12191368948550384, + "25_to_34": 0.3111513455503956, + "35_to_49": 0.37171601251334024, + "50_to_61": 0.13719732675785898, + "62_and_older": 0.05802162569290144 + }, + "graduate_share_of_own_original_principal": 0.4995084468534534, + "ibr_new_borrower_share": 0.6132625665271612 + } + }, + "metrics_vs_scf": { + "share_with_debt_error_pp": -1.7412858119937007, + "mean_among_borrowers_error_pct": 3.965667719599386, + "median_among_borrowers_error_pct": -7.9999999999999964, + "household_balance_band_tvd": 0.046105502753231414, + "head_age_band_share_mae_pp": 2.3908244112012222, + "race_share_mae_pp": 0.9415277758683394, + "income_quintile_share_mae_pp": 2.865766910648814, + "borrower_balance_band_tvd": 0.0457178698849765, + "borrower_age_band_tvd": 0.04223818791519181 + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 56f39195cf..7810324f4a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_404 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_184 +EXPECTED_AUTHORED_FIELD_COUNT = 32_584 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_784 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_368 class FieldUsageError(AssertionError): @@ -361,8 +361,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "36564747627fb0058bb99e8e8c4397e41d8ecc472a20c49d4b6d7df1003a80ec", ), "calibration": ( - 312, - "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + 316, + "0212e52e2bf1e5996fe6b4785b2d74f2fb5680ec38b258c6ad4dbeadbd181a1b", ), "catalog_columns": ( 1_673, @@ -461,8 +461,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", ), "resolved_vintage_authorities": ( - 63, - "5f2edfe9826e1200e2b8706780c9f53b7c61f01aad8cab5907f08c37a0b1ead2", + 67, + "3f49902e745368ce8df0ad338a1697b76fd3299d35959977d36a8c09690ffb10", ), "selection": ( 87, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_716, - "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + 1_884, + "886eca845bf2e67788b19c28aa24459b9e3a5edb5f2d8129fb47d13804eca393", ), "spine_assembly_household_mass_shares": ( 2, @@ -533,8 +533,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d53096196db4c34da260cce2f35af8e7ba67f978448656c602ee5e17529dc4e0", ), "vintage_records": ( - 112, - "c69a923e893fde8a929ca087d29ded607fa9c04fd14a39d0a401ad1db845e6de", + 120, + "825b4ab09602353fc4d6c24b6f7654f7582502c7acf963b7ff6c117b07208eb4", ), } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 4a20e69687..03efc9093e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "0af421c69e29c23eedd42b1d1e890b8223d637101b8b681e8fa0d07124591b72", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", + "late_resource_semantics": "429406ee38dbdcf5d751a86e37004636931b69f772e346fd8c853510ba54b05d", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -369,9 +369,9 @@ # not run, changes its probes, so again no US solve changes. "seed_map": "0db599df419dd16779097361dc72afdefb8df2abf8b134078da9c52d31d8e017", "seed_protocol": "2b7bdc1d6b0da1bdaf54b4cac29f10f2d3c8bed2e177729c2c89e6b33a099cc5", - "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "source_manifest": "ef7ed58e0170f21db8be7812f99251e702bb6b429aae463a80ea0a870c16ced5", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", - "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", + "tail": "b3e7289f8107138e23e8d7c9a172285afd2099d45558d253cc369eebe8ba3406", } INVENTORY_REPORT_SCHEMA_VERSION = 1 @@ -441,7 +441,7 @@ "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -451,7 +451,7 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, @@ -1419,7 +1419,7 @@ def add( add( "capital_gains_tail_contract_exact", clauses={ - "tail terminal field count differs": tail_count == 934, + "tail terminal field count differs": tail_count == 938, "resolved calibration tails differ": _json_equal( legacy.get("calibration_tail_contracts"), expected_legacy["calibration_tail_contracts"], @@ -1447,7 +1447,7 @@ def add( "legacy_adapter.stacked_checkpoint_static_components", ), observed={"terminal_fields": tail_count, "sha256": sha256_json(tail_bundle)}, - expected={"terminal_fields": 934, "sha256": EXPECTED_HASHES["tail"]}, + expected={"terminal_fields": 938, "sha256": EXPECTED_HASHES["tail"]}, ) add( diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 72b56c12ab..ebefe0ee1b 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -855,7 +855,7 @@ }, { "stage": "scf_wealth", - "survey": "Fed SCF 2022 + Census SIPP 2023", + "survey": "Fed SCF 2022 + Fed SCF 2025 + Census SIPP 2023", "source": "https://www.federalreserve.gov/econres/scfindex.htm", "grain": "household", "artifacts": [ @@ -895,6 +895,26 @@ "revision": "21280dca5995e978d706740a8a4b9b7860cfd7b6", "sha256": "5c30439e365fc26483318ef61d1d8f4bb2f0e9d6bb47c22c06756a7698733ee2", "size_bytes": 3726010471 + }, + { + "kind": "public_microdata", + "format": "stata_in_zip", + "vintage": "2025", + "locator": "https://www.federalreserve.gov/econres/files/scfp2025s.zip", + "member": "rscfp2025.dta", + "sha256": "32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + "member_sha256": "454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd", + "expected_rows": 21800 + }, + { + "kind": "public_microdata", + "format": "stata_in_zip", + "vintage": "2025", + "locator": "https://www.federalreserve.gov/econres/files/scf2025s.zip", + "member": "p25i6.dta", + "sha256": "c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + "member_sha256": "38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad", + "expected_rows": 21800 } ], "operations": [ @@ -1149,6 +1169,186 @@ "stock_assets" ] }, + { + "kind": "read_tables", + "tables": [ + "scf_2025_summary_extract", + "scf_2025_full_extract" + ], + "weight": "wgt" + }, + { + "kind": "join", + "left": "scf_2025_summary_extract", + "right": "scf_2025_full_extract", + "keys": [ + "y1", + "yy1" + ], + "validation": "one_to_one_all_five_implicates" + }, + { + "kind": "derive", + "outputs": [ + "federal_education_debt", + "head_federal_student_loan_balance", + "spouse_federal_student_loan_balance", + "child_federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "parent_plus_share", + "child_own_share", + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "child_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower", + "adult_children_in_household" + ], + "method": "scf_federal_education_loans_by_holder", + "inputs": [ + "x7824", + "x7847", + "x7870", + "x7924", + "x7947", + "x7970", + "x7179", + "x7879", + "x7884", + "x7889", + "x7894", + "x7899", + "x7994", + "x7978", + "x7883", + "x7888", + "x7893", + "x7898", + "x7993", + "x9332", + "x9333", + "x9334", + "x9335", + "x9336", + "x9337", + "x9338", + "x9339", + "x9340", + "x9341", + "x9342", + "x9343", + "x7805", + "x7828", + "x7851", + "x7905", + "x7928", + "x7951", + "x8000", + "x108", + "x114", + "x120", + "x126", + "x132", + "x202", + "x208", + "x214", + "x220", + "x226", + "x110", + "x116", + "x122", + "x128", + "x134", + "x204", + "x210", + "x216", + "x222", + "x228" + ], + "allocation": "x7179_in_proportion_to_loans_1_6", + "source": "scf_2025_full_extract" + }, + { + "kind": "fit_weighted_qrf", + "donor": "scf_2025_education_debt", + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income" + ], + "targets": [ + "federal_education_debt" + ], + "weight": "wgt", + "n_estimators": 100, + "seed_from_build_config": true + }, + { + "kind": "fit_weighted_qrf", + "donor": "scf_2025_education_debt_borrowers", + "predictors": [ + "age", + "is_female", + "cps_race", + "is_married", + "own_children_in_household", + "employment_income", + "household_income", + "adult_children_in_household", + "federal_education_debt" + ], + "targets": [ + "parent_plus_share", + "child_own_share", + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "child_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower" + ], + "weight": "wgt", + "response_filter": "federal_education_debt > 0", + "n_estimators": 100, + "seed_from_build_config": true + }, + { + "kind": "derive", + "outputs": [ + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share" + ], + "method": "allocate_household_education_debt_to_reference_person_spouse_and_adult_child", + "inputs": [ + "federal_education_debt", + "parent_plus_share", + "child_own_share", + "head_own_share", + "spouse_own_share", + "head_graduate_share", + "spouse_graduate_share", + "child_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower", + "A_SPOUSE", + "PECOHAB", + "PEPAR1", + "PEPAR2", + "age" + ] + }, { "kind": "support_clip", "range": "donor_realized" @@ -1180,7 +1380,11 @@ "qualified_passenger_vehicle_loan_interest", "bank_account_assets", "bond_assets", - "stock_assets" + "stock_assets", + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share" ], "nonnegative_outputs": [ "auto_loan_balance", @@ -1188,9 +1392,12 @@ "qualified_passenger_vehicle_loan_interest", "bank_account_assets", "bond_assets", - "stock_assets" + "stock_assets", + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "federal_student_loan_graduate_share" ], - "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped." + "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans 1-6): X7179 takes the family's federal share and holder split of loans 1-6. Whose education: code 1 (the reference person) and code 2 (spouse or partner) are each person's own federal_student_loan_balance. Where X8000 = 1 the public file reverses respondent and spouse/partner codes, but not those of loans 4 and 6 (X7893, X7993): completed degree loans consistent with exactly one partner's highest degree (X5931, X6111) match the coded person on loans 1, 2, 3 and 5 and on loans 4 and 6 of unreversed families (25 of 25), but on loans 4 and 6 of reversed families match in none of 54 (all implicates), so codes 1 and 2 of those two loans are swapped back in reversed families. Code 3 (child, with grandchild and other relative folded in by the public file) is a co-resident child's own federal_student_loan_balance when the household listing (X108-X226) has a child aged 18 or over who could have been in school in the year borrowed (16 or older then, at the oldest age of the public file's age band), unless only a parent could have borrowed it: an undergraduate or certificate loan above the $31,000 aggregate limit on a dependent undergraduate's own Direct Loans (34 CFR 685.203(e)(1)) borrowed while the child could still have been under 24 and so dependent (20 U.S.C. 1087vv(d)(1)(A)). Graduate loans and loans borrowed after the child turned 24 are never Parent PLUS, which finances only a dependent undergraduate (34 CFR 685.200(c)(2)(i)). Every other code-3 loan is the reference person's federal_parent_plus_loan_balance: the screener (X7801) leaves out loans of people living elsewhere, so a loan for a child not in the household is the parent's. Codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the reference person's own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower is true when every federal loan the person holds (for the reference person, Parent PLUS too, as 34 CFR 685.209(b)(13)(ii) looks at any Direct or FFEL balance) was borrowed in 2014 or later (the item records a year only, so 2014 counts as after July 1, 2014); the share is an upper bound, because paid-off pre-2014 loans are not observed and a consolidated loan is dated by its most recent underlying loan. federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person's own federal loans that financed a master's, professional or doctoral program (codes 13-15). One weighted QRF draws the household total from all families; a second, fit on borrower families and conditioning also on the total and the number of adult children, draws the Parent PLUS, adult child's, reference person's and spouse's shares in that order, normalized to sum to one, then the three graduate shares and new-borrower flags. Both condition on the reference person's age, sex and race, a spouse or partner being present (SCF MARRIED counts partners), the children of the reference person or partner (SCF KIDS), and the wages and income less capital gains of the reference person, spouse or partner and their children (the CPS counterpart of the SCF primary economic unit). The shared interest-dividend and Social Security-pension predictors are left out: the CPS host carries tax-record imputations of both, so at reference ages 25-34 42% and 18% of host households report some against 16% and 6% of SCF families, and mapping them to the SCF distribution moves the host's borrower share most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). The reference person is the CPS householder or, with a spouse or partner present, the man of a mixed-sex couple or else the older partner; that person, the member named by its A_SPOUSE (else PECOHAB) pointer, and the youngest child of either aged 18 or over (PEPAR1/PEPAR2) receive the balances. Without a spouse or partner present the spouse's draw goes to the reference person, and a child's draw in a household without an adult child is the reference person's Parent PLUS loan. Balances are 2025 survey-date dollars; the FSA calibration (microcosm#1074) pins totals." }, { "stage": "ssi_disability_criteria", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml index 128371e097..b73b86ef0e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml @@ -278,6 +278,9 @@ tail_contracts: - farm_operations_income_would_be_qualified - farm_rent_income - farm_rent_income_would_be_qualified + - federal_parent_plus_loan_balance + - federal_student_loan_balance + - federal_student_loan_graduate_share - first_home_mortgage_balance - first_home_mortgage_interest - first_home_mortgage_origination_year @@ -292,6 +295,7 @@ tail_contracts: - is_executive_administrative_professional - is_farmer_fisher - is_hispanic + - is_income_based_repayment_new_borrower - is_military - is_paid_hourly - is_union_member_or_covered diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index e33ffec848..4c08305587 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf + sha256: 5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b stage_manifest: version: 1 country: us @@ -954,7 +954,7 @@ stages: nonnegative_outputs: - schedule_d_capital_gain_distributions - stage: scf_wealth - survey: Fed SCF 2022 + Census SIPP 2023 + survey: Fed SCF 2022 + Fed SCF 2025 + Census SIPP 2023 source: https://www.federalreserve.gov/econres/scfindex.htm grain: household artifacts: @@ -989,6 +989,22 @@ stages: revision: 21280dca5995e978d706740a8a4b9b7860cfd7b6 sha256: 5c30439e365fc26483318ef61d1d8f4bb2f0e9d6bb47c22c06756a7698733ee2 size_bytes: 3726010471 + - kind: public_microdata + format: stata_in_zip + vintage: '2025' + locator: https://www.federalreserve.gov/econres/files/scfp2025s.zip + member: rscfp2025.dta + sha256: 32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb + member_sha256: 454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd + expected_rows: 21800 + - kind: public_microdata + format: stata_in_zip + vintage: '2025' + locator: https://www.federalreserve.gov/econres/files/scf2025s.zip + member: p25i6.dta + sha256: c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741 + member_sha256: 38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad + expected_rows: 21800 operations: - kind: read_tables tables: @@ -1200,6 +1216,164 @@ stages: - bank_account_assets - bond_assets - stock_assets + - kind: read_tables + tables: + - scf_2025_summary_extract + - scf_2025_full_extract + weight: wgt + - kind: join + left: scf_2025_summary_extract + right: scf_2025_full_extract + keys: + - y1 + - yy1 + validation: one_to_one_all_five_implicates + - kind: derive + outputs: + - federal_education_debt + - head_federal_student_loan_balance + - spouse_federal_student_loan_balance + - child_federal_student_loan_balance + - federal_parent_plus_loan_balance + - parent_plus_share + - child_own_share + - head_own_share + - spouse_own_share + - head_graduate_share + - spouse_graduate_share + - child_graduate_share + - head_is_ibr_new_borrower + - spouse_is_ibr_new_borrower + - child_is_ibr_new_borrower + - adult_children_in_household + method: scf_federal_education_loans_by_holder + inputs: + - x7824 + - x7847 + - x7870 + - x7924 + - x7947 + - x7970 + - x7179 + - x7879 + - x7884 + - x7889 + - x7894 + - x7899 + - x7994 + - x7978 + - x7883 + - x7888 + - x7893 + - x7898 + - x7993 + - x9332 + - x9333 + - x9334 + - x9335 + - x9336 + - x9337 + - x9338 + - x9339 + - x9340 + - x9341 + - x9342 + - x9343 + - x7805 + - x7828 + - x7851 + - x7905 + - x7928 + - x7951 + - x8000 + - x108 + - x114 + - x120 + - x126 + - x132 + - x202 + - x208 + - x214 + - x220 + - x226 + - x110 + - x116 + - x122 + - x128 + - x134 + - x204 + - x210 + - x216 + - x222 + - x228 + allocation: x7179_in_proportion_to_loans_1_6 + source: scf_2025_full_extract + - kind: fit_weighted_qrf + donor: scf_2025_education_debt + predictors: + - age + - is_female + - cps_race + - is_married + - own_children_in_household + - employment_income + - household_income + targets: + - federal_education_debt + weight: wgt + n_estimators: 100 + seed_from_build_config: true + - kind: fit_weighted_qrf + donor: scf_2025_education_debt_borrowers + predictors: + - age + - is_female + - cps_race + - is_married + - own_children_in_household + - employment_income + - household_income + - adult_children_in_household + - federal_education_debt + targets: + - parent_plus_share + - child_own_share + - head_own_share + - spouse_own_share + - head_graduate_share + - spouse_graduate_share + - child_graduate_share + - head_is_ibr_new_borrower + - spouse_is_ibr_new_borrower + - child_is_ibr_new_borrower + weight: wgt + response_filter: federal_education_debt > 0 + n_estimators: 100 + seed_from_build_config: true + - kind: derive + outputs: + - federal_student_loan_balance + - federal_parent_plus_loan_balance + - is_income_based_repayment_new_borrower + - federal_student_loan_graduate_share + method: allocate_household_education_debt_to_reference_person_spouse_and_adult_child + inputs: + - federal_education_debt + - parent_plus_share + - child_own_share + - head_own_share + - spouse_own_share + - head_graduate_share + - spouse_graduate_share + - child_graduate_share + - head_is_ibr_new_borrower + - spouse_is_ibr_new_borrower + - child_is_ibr_new_borrower + - A_SPOUSE + - PECOHAB + - PEPAR1 + - PEPAR2 + - age - kind: support_clip range: donor_realized outputs: @@ -1229,6 +1403,10 @@ stages: - bank_account_assets - bond_assets - stock_assets + - federal_student_loan_balance + - federal_parent_plus_loan_balance + - is_income_based_repayment_new_borrower + - federal_student_loan_graduate_share nonnegative_outputs: - auto_loan_balance - auto_loan_interest @@ -1236,6 +1414,9 @@ stages: - bank_account_assets - bond_assets - stock_assets + - federal_student_loan_balance + - federal_parent_plus_loan_balance + - federal_student_loan_graduate_share notes: 'Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45''s final 13-predictor asset QRF, while @@ -1251,7 +1432,52 @@ stages: leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS''s approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth - is never clipped.' + is never clipped. Federal education debt (microcosm#1073) reads the 2025 SCF while every other output + stays on 2022. For education loans 1-6 the full file records the amount owed, the federal flag (X7879/X7884/X7889/X7894/X7899/X7994 + = 1), whose education the loan paid for (X7978/X7883/X7888/X7893/X7898/X7993), the year borrowed (X9332-X9337), + the program (X9338-X9343) and the amount borrowed (X7805/X7828/X7851/X7905/X7928/X7951); only federal + loans count, and a blank federal flag counts as non-federal. X7179, the total owed on loans beyond + the sixth, carries none of these items, so every loan 1-6 is scaled by 1 + X7179 / (balance on loans + 1-6): X7179 takes the family''s federal share and holder split of loans 1-6. Whose education: code + 1 (the reference person) and code 2 (spouse or partner) are each person''s own federal_student_loan_balance. + Where X8000 = 1 the public file reverses respondent and spouse/partner codes, but not those of loans + 4 and 6 (X7893, X7993): completed degree loans consistent with exactly one partner''s highest degree + (X5931, X6111) match the coded person on loans 1, 2, 3 and 5 and on loans 4 and 6 of unreversed families + (25 of 25), but on loans 4 and 6 of reversed families match in none of 54 (all implicates), so codes + 1 and 2 of those two loans are swapped back in reversed families. Code 3 (child, with grandchild and + other relative folded in by the public file) is a co-resident child''s own federal_student_loan_balance + when the household listing (X108-X226) has a child aged 18 or over who could have been in school in + the year borrowed (16 or older then, at the oldest age of the public file''s age band), unless only + a parent could have borrowed it: an undergraduate or certificate loan above the $31,000 aggregate + limit on a dependent undergraduate''s own Direct Loans (34 CFR 685.203(e)(1)) borrowed while the child + could still have been under 24 and so dependent (20 U.S.C. 1087vv(d)(1)(A)). Graduate loans and loans + borrowed after the child turned 24 are never Parent PLUS, which finances only a dependent undergraduate + (34 CFR 685.200(c)(2)(i)). Every other code-3 loan is the reference person''s federal_parent_plus_loan_balance: + the screener (X7801) leaves out loans of people living elsewhere, so a loan for a child not in the + household is the parent''s. Codes 6 (multiple people), 7 (unrelated person) and -7 (other) are the + reference person''s own balance, since none can be a Parent PLUS loan for one dependent student. is_income_based_repayment_new_borrower + is true when every federal loan the person holds (for the reference person, Parent PLUS too, as 34 + CFR 685.209(b)(13)(ii) looks at any Direct or FFEL balance) was borrowed in 2014 or later (the item + records a year only, so 2014 counts as after July 1, 2014); the share is an upper bound, because paid-off + pre-2014 loans are not observed and a consolidated loan is dated by its most recent underlying loan. + federal_student_loan_graduate_share is the amount-borrowed-weighted share of the person''s own federal + loans that financed a master''s, professional or doctoral program (codes 13-15). One weighted QRF + draws the household total from all families; a second, fit on borrower families and conditioning also + on the total and the number of adult children, draws the Parent PLUS, adult child''s, reference person''s + and spouse''s shares in that order, normalized to sum to one, then the three graduate shares and new-borrower + flags. Both condition on the reference person''s age, sex and race, a spouse or partner being present + (SCF MARRIED counts partners), the children of the reference person or partner (SCF KIDS), and the + wages and income less capital gains of the reference person, spouse or partner and their children + (the CPS counterpart of the SCF primary economic unit). The shared interest-dividend and Social Security-pension + predictors are left out: the CPS host carries tax-record imputations of both, so at reference ages + 25-34 42% and 18% of host households report some against 16% and 6% of SCF families, and mapping them + to the SCF distribution moves the host''s borrower share most of any predictor (experiments/receipts/1073-predictor-shift-diagnosis.json). + The reference person is the CPS householder or, with a spouse or partner present, the man of a mixed-sex + couple or else the older partner; that person, the member named by its A_SPOUSE (else PECOHAB) pointer, + and the youngest child of either aged 18 or over (PEPAR1/PEPAR2) receive the balances. Without a spouse + or partner present the spouse''s draw goes to the reference person, and a child''s draw in a household + without an adult child is the reference person''s Parent PLUS loan. Balances are 2025 survey-date + dollars; the FSA calibration (microcosm#1074) pins totals.' - stage: ssi_disability_criteria survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml index a5b70b9085..fc2ca10fad 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/vintages.yaml @@ -110,6 +110,15 @@ records: field: vintage compatible_with: - vintage:target_2024 +- id: scf_2025 + kind: survey_period_ref + authority_ref: + kind: source_stage_artifact + stage: scf_wealth + artifact_index: 3 + field: vintage + compatible_with: + - vintage:target_2024 - id: sipp_2023 kind: survey_period_ref authority_ref: @@ -134,6 +143,7 @@ records: - vintage:policyengine_us_surface - vintage:release_us_2024 - vintage:scf_2022 + - vintage:scf_2025 - vintage:sipp_2023 - vintage:tax_2015 - id: tax_2015 diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 74e4757cba..1487763de7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -764,6 +764,28 @@ us_scf_auto_loans_summary, with_us_scf_auto_loan_inputs, ) +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + SCF_EDUCATION_DEBT_VINTAGE, + SCF_WHOSE_EDUCATION_HOLDER, + US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS, + US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, + derive_scf_education_debt_targets, + fetch_scf_public_file, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, + us_scf_education_debt_stage_spec, + us_scf_education_debt_summary, + with_us_scf_education_debt_inputs, +) from microcosm.build.us_runtime.scf_wealth import ( FINANCIAL_ASSET_BLEND_AUDIT_KEY, FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY, @@ -1575,6 +1597,26 @@ "us_scf_auto_loans_stage_spec", "us_scf_auto_loans_summary", "with_us_scf_auto_loan_inputs", + "SCF_2025_FULL_EXTRACT", + "SCF_2025_SUMMARY_EXTRACT", + "SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS", + "SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS", + "SCF_EDUCATION_DEBT_PREDICTORS", + "SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION", + "SCF_EDUCATION_DEBT_VINTAGE", + "SCF_WHOSE_EDUCATION_HOLDER", + "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", + "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS", + "derive_scf_education_debt_targets", + "fetch_scf_public_file", + "impute_us_scf_education_debt", + "load_scf_2025_education_debt_donor", + "scf_education_debt_donor_fit_table", + "us_scf_education_debt_fit_table", + "us_scf_education_debt_signal_gate", + "us_scf_education_debt_stage_spec", + "us_scf_education_debt_summary", + "with_us_scf_education_debt_inputs", "FINANCIAL_ASSET_BLEND_AUDIT_KEY", "FINANCIAL_ASSET_SOURCE_SCF_PROBABILITY", "SCF_FINANCIAL_ASSET_TARGET_COMPONENTS", @@ -2082,13 +2124,14 @@ def to_manifest(self) -> dict[str, object]: #: the observatory's sources diagram and the dataset card derive from it. US_DONORS: Mapping[str, DonorSpec] = { "scf_wealth": DonorSpec( - survey="Fed SCF 2022 + Census SIPP 2023", + survey="Fed SCF 2022 + Fed SCF 2025 + Census SIPP 2023", source="https://www.federalreserve.gov/econres/scfindex.htm", notes=( "SCF anchors signed net worth and one half of household liquid-asset " "vectors; the immutable SIPP 2023 public-use donor supplies the " "other half and restores low liquid-asset mass. Auto loans use the " - "full SCF separately." + "full SCF separately. Federal education debt reads the 2025 SCF " + "(microcosm#1073) while every other output stays on 2022." ), ), US_SSI_DISABILITY_CRITERIA_STAGE_NAME: DonorSpec( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py new file mode 100644 index 0000000000..dc12245ef2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_education_debt.py @@ -0,0 +1,2333 @@ +"""Federal student loan balances from the 2025 Survey of Consumer Finances. + +microcosm#1073 carries SCF education debt into the US build for the student debt +pilot (PolicyEngine/us-student-debt#1). It writes four person inputs of +PolicyEngine-US: ``federal_student_loan_balance`` and +``federal_parent_plus_loan_balance`` (policyengine-us#9721), and +``is_income_based_repayment_new_borrower`` and +``federal_student_loan_graduate_share`` (policyengine-us#9724). + +**Vintage.** Only education debt reads the 2025 SCF. Net worth, the liquid-asset +leaves and auto loans stay on the 2022 SCF in :mod:`scf_wealth` and +:mod:`scf_auto_loans` until the whole ``scf_wealth`` stage moves. The 2025 wave +is three years closer to the build, follows the end of the payment pause, and +records each loan's year borrowed and program type, which 2022 does not. Every +2025-specific coordinate lives in :data:`SCF_2025_SUMMARY_EXTRACT` and +:data:`SCF_2025_FULL_EXTRACT`, and the predictor construction is the shared +summary-extract one from :mod:`scf_wealth`. The 2025 summary extract carries +every column that construction reads, so moving the stage is a change of pinned +coordinates, not of code. + +**Donor targets, per SCF family and implicate.** For each of education loans +1-6 the full public file records the amount still owed (X7824, X7847, X7870, +X7924, X7947, X7970), whether it is a federal loan (X7879, X7884, X7889, X7894, +X7899, X7994; 1 = yes), whose education it paid for (X7978, X7883, X7888, +X7893, X7898, X7993), the year borrowed (X9332-X9337), the program it financed +(X9338-X9343) and the amount borrowed (X7805, X7828, X7851, X7905, X7928, +X7951). Loans beyond the sixth are one total, X7179, with none of these items. + +- Only loans flagged federal (code 1) count. A loan whose federal status is + not recorded (code 0; the 2025 public file leaves some "don't know" answers + blank) counts as non-federal. +- **X7179 rule.** The remaining-loans total is allocated in proportion to the + family's balances on loans 1-6: every loan 1-6 is scaled by + ``1 + X7179 / (sum of balances on loans 1-6)``. This applies the family's + federal share of loans 1-6 to X7179, and splits the federal part among people + in the same proportions as loans 1-6. A family with X7179 but no balance on + loans 1-6 contributes nothing (none in the public files read). +- **Whose-education rule** (:data:`SCF_WHOSE_EDUCATION_HOLDER`). Code 1 + (respondent) is the head's own loan and code 2 (spouse or partner) the + spouse's; both go to ``federal_student_loan_balance``. Where X8000 = 1 the + public file reverses every respondent and spouse/partner variable so that + code 1 is the SCF reference person, but it left the codes of loans 4 and 6 + (X7893, X7993) unreversed. The test: a completed loan for a degree should + be consistent with the coded person's highest degree (X5931 for the + reference person, X6111 for the spouse or partner). Of the loans consistent + with exactly one partner's degree, loans 1, 2, 3 and 5 match the coded person + in reversed and other families alike, and loans 4 and 6 match in all 25 such + loans (five implicates) of families that were not reversed, but in none of + the 54 of reversed families (10 in implicate 1). So in a reversed family + codes 1 and 2 of loans 4 and 6 are swapped back + (:data:`SCF_WHOSE_EDUCATION_UNREVERSED_LOANS`); the receipt reruns the test. + The codebook names no such exception, so the correction is tied to the + pinned member SHA-256. Code 6 (multiple people), code 7 (unrelated person) + and code -7 (other) go to the head's own balance: none can be a Parent PLUS + loan for one dependent student, and the SCF reference person is the + household's default borrower. Any other code (the public file folds 4 and 5 + into 3, so none is expected) also goes to the head. +- **Loans for a child's education** (code 3, child or stepchild; the public + file folds grandchild and other relative into it). The survey asks about + the loans of the family living in the household, and the screener (X7801) + leaves out loans of people who live elsewhere, so a code-3 loan is a + co-resident child's own loan or the parent's Parent PLUS loan. It is the + child's own (the child's ``federal_student_loan_balance``) when the + household listing (persons 3-12, X108-X226) has a child (code 4) aged 18 or + over who could have been in school when it was borrowed (16 or older in + that year, at the oldest age the child's public-file age band allows: the + file bands listed ages, so 18-27 all read 25), unless only a parent could + have borrowed it: an undergraduate or certificate loan whose amount + borrowed exceeds the $31,000 aggregate limit on a dependent undergraduate's + own Direct Loans (34 CFR 685.203(e)(1)), borrowed while the child could + still have been under 24, and so dependent (20 U.S.C. 1087vv(d)(1)(A)). + Parent PLUS finances only a dependent undergraduate (34 CFR + 685.200(c)(2)(i)), so a graduate loan, or one borrowed after every fitting + child turned 24, is never Parent PLUS. Every other code-3 loan is the + head's ``federal_parent_plus_loan_balance``: no listed adult child could + have borrowed it, so the student lives elsewhere and only the parent's loan + is reported, or it exceeds the dependent limit. An undergraduate loan + within the limit for a child of dependent age could be either, and is read + as the child's own, the household member the listing names; the receipt + reports the Parent PLUS share under the other reading too. +- **IBR new-borrower status** is true when the person holds a federal loan and + every federal loan the person holds (own, and for the head Parent PLUS too, + since 34 CFR 685.209(b)(13)(ii) looks at any outstanding Direct or FFEL + balance) was borrowed in 2014 or later. The item records a year only, so + 2014 counts as after July 1, 2014. The share is an upper bound: the SCF sees + only loans still owed, so a borrower whose pre-2014 loans have been paid + off reads as new, and a consolidated loan is dated by its most recent + underlying loan, so a consolidation of pre-2014 loans can read as new too. +- **Graduate share** is the share of the original amount of the person's own + federal loans (Parent PLUS excluded) that financed a master's, professional + or doctoral program (codes 13-15), weighted by the amount borrowed (falling + back to the balance when the amount is missing), as 34 CFR 685.209(f)(1) + weights the SAVE rates. + +**Imputation.** Two regime-gated weighted QRFs (:class:`microcosm.fit.QRF`) +condition on the same predictors (:data:`SCF_EDUCATION_DEBT_PREDICTORS`): the +reference person's age, sex and race, whether a spouse or partner is present, +the number of children, wages, and income excluding capital gains. The first +draws each recipient household's federal education debt from every SCF family. +The second, fit on borrower families only, conditions also on that total and +on the number of adult children (children of the reference person or partner +aged 18 or over), and draws the chained composition: the Parent PLUS, adult +child's, head's and spouse's shares of the total, in that order, then the +three graduate shares and the three new-borrower flags. The four shares are +clipped to [0, 1] and normalized to sum to one, and a borrower drawn with no +share holds its own loan. Parent PLUS is therefore drawn for itself, first: a +family that resembles donors without Parent PLUS draws none, instead of +receiving whatever the own shares leave. + +Two of the shared SCF predictors are left out +(:data:`SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS` keeps the full list for the +diagnostics). Interest and dividends, and Social Security and pensions, are +measured differently on the two sides: the CPS host carries tax-record +imputations, the SCF survey reports. At reference ages 25-34, 42% of host +households have interest or dividends against 16% of SCF families, and 18% +have Social Security or pension income against 6%. A QRF reads equal values +as equal positions, so the host's older-looking households drew less debt: +mapping these two predictors to their SCF distribution at equal rank moves the +host's borrower share most of any predictor +(``experiments/receipts/1073-predictor-shift-diagnosis.json``). Wages and +total income stay in, because the issue asks for income and they carry the +income gradient. + +**Recipients** (:data:`SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION`). The CPS side +builds each predictor the way the SCF defines it. The reference person is the +CPS householder or, when a spouse or cohabiting partner is present, the man of +a mixed-sex couple or else the older partner: the SCF's ordering of its +economically dominant couple. ``is_married`` counts a cohabiting partner, as +SCF ``MARRIED`` does, and children count those of the reference person or +partner, as SCF ``KIDS`` does; adult children are the ones aged 18 or over, +as listing code 4 aged 18 or over counts them. Incomes sum over the reference +person, spouse or partner and their children: the SCF primary economic unit, +which leaves out a financially independent co-resident relative or roommate. +The spouse is the member whose line number is the reference person's +``A_SPOUSE``, or else its ``PECOHAB`` partner when that column is present. A +household drawn with a spouse's balance but no spouse or partner present +gives that balance to the reference person, whose graduate share becomes the +balance-weighted mean of the two and whose new-borrower flag requires both. +The child's draw goes to the youngest child of the reference person or +partner aged 18 or over (``PEPAR1``/``PEPAR2``). A household without one +cannot use it, and the draw makes it the head's Parent PLUS loan, the SCF +side's rule for a loan whose student lives elsewhere. Everyone else gets zero. +:data:`SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION` keeps the first receipt's +construction (oldest adult, ``A_MARITL`` marriage, own children, whole +household) for its before/after table. + +**Known limits.** The SCF asks about the loans of the family living in the +household (its primary economic unit), so a financially independent adult in +someone else's household is missed unless the family reports the loan as one +for a child's education. No education-loan item names the listed person a +loan financed, and the public file bands listed ages, so the child rule works +from age bands, and with two or more adult children the youngest holds the +loan. The SCF federal flag includes FFEL and Perkins loans that +``federal_student_loan_balance`` excludes until consolidated. Balances are +2025 survey-date dollars. The FSA calibration (microcosm#1074) pins totals. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.gates import GateResult +from microcosm.build.source_manifest import SourceStageSpec, load_source_manifest +from microcosm.build.us_runtime.eligibility_inputs import _own_children_in_household +from microcosm.build.us_runtime.scf_auto_loans import _scf_reference_person_mask +from microcosm.build.us_runtime.scf_wealth import ( + SCF_WEALTH_PREDICTORS, + _recipient_cps_race, + _replace_sentinels, + _scf_summary_predictor_table, + _sum_present, +) +from microcosm.frame import Frame +from microcosm.frame.units import US_SCHEMA + +__all__ = [ + "SCF_2025_FULL_EXTRACT", + "SCF_2025_SUMMARY_EXTRACT", + "SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR", + "SCF_EDUCATION_DEBT_BALANCE_BANDS", + "SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS", + "SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS", + "SCF_EDUCATION_DEBT_COMPOSITION_TARGETS", + "SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS", + "SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION", + "SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS", + "SCF_EDUCATION_DEBT_PREDICTORS", + "SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION", + "SCF_EDUCATION_DEBT_TOTAL_TARGET", + "SCF_EDUCATION_DEBT_VINTAGE", + "SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS", + "SCF_EDUCATION_LOAN_BALANCE_COLUMNS", + "SCF_EDUCATION_LOAN_FEDERAL_COLUMNS", + "SCF_EDUCATION_LOAN_PROGRAM_COLUMNS", + "SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN", + "SCF_EDUCATION_LOAN_WHOSE_COLUMNS", + "SCF_EDUCATION_LOAN_YEAR_COLUMNS", + "SCF_HOUSEHOLD_LISTING_AGE_COLUMNS", + "SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS", + "SCF_REVERSED_COLUMN", + "SCF_WHOSE_EDUCATION_HOLDER", + "SCF_WHOSE_EDUCATION_UNREVERSED_LOANS", + "US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS", + "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS", + "RecipientDefinition", + "ScfPublicFile", + "derive_scf_education_debt_targets", + "draw_scf_education_debt_households", + "education_debt_fit_table", + "fetch_scf_public_file", + "impute_us_scf_education_debt", + "load_scf_2025_education_debt_donor", + "scf_education_debt_composition_predictors", + "scf_education_debt_donor_fit_table", + "us_scf_education_debt_fit_table", + "us_scf_education_debt_signal_gate", + "us_scf_education_debt_stage_spec", + "us_scf_education_debt_summary", + "with_us_scf_education_debt_inputs", +] + +_SCF_STAGE_NAME = "scf_wealth" +_DONOR_WEIGHT_COLUMN = "scf_weight" +_HOUSEHOLD_ID_COLUMN = "person_household_id" +_DEFAULT_N_ESTIMATORS = 100 +_SCF_HTTP_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" +) + +#: The SCF wave education debt reads (``scf_wealth`` itself stays on 2022). +SCF_EDUCATION_DEBT_VINTAGE = "2025" + + +@dataclass(frozen=True) +class ScfPublicFile: + """One pinned Federal Reserve SCF public file: a Stata member in a zip.""" + + url: str + member: str + zip_sha256: str + member_sha256: str + rows: int + + @property + def zip_name(self) -> str: + return self.url.rsplit("/", maxsplit=1)[-1] + + +#: The 2025 public summary extract. Both 2025 files were released 2026-10-09; +#: these digests match a fresh Federal Reserve download of 2026-10-12. Each +#: file has one row per family and implicate (4,360 families x 5). +SCF_2025_SUMMARY_EXTRACT = ScfPublicFile( + url="https://www.federalreserve.gov/econres/files/scfp2025s.zip", + member="rscfp2025.dta", + zip_sha256="32dcf86be48b20fb5fd9e659f79fa22a4d9cccf456e44b6ca9b1329e9869bbeb", + member_sha256="454a287f188bf13f94efc09820fa878482d897db1d2a16ac63bec14eb580d2dd", + rows=21_800, +) +#: The 2025 full public file. +SCF_2025_FULL_EXTRACT = ScfPublicFile( + url="https://www.federalreserve.gov/econres/files/scf2025s.zip", + member="p25i6.dta", + zip_sha256="c4010bd5decaa13d5d027862ed4b6c41b4106925089290f879749f8f5a467741", + member_sha256="38926275dfb8f00ea843b80ced7a9412ad4653d39c7fab8fe24c3bdf912902ad", + rows=21_800, +) + +#: Education loans 1-6 (codebook "EDUCATION LOANS"): amount still owed. +SCF_EDUCATION_LOAN_BALANCE_COLUMNS: tuple[str, ...] = ( + "x7824", + "x7847", + "x7870", + "x7924", + "x7947", + "x7970", +) +#: Total owed on all remaining loans beyond the sixth. +SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN = "x7179" +#: "Is this loan a federal student loan?" 1 = yes, 5 = no. +SCF_EDUCATION_LOAN_FEDERAL_COLUMNS: tuple[str, ...] = ( + "x7879", + "x7884", + "x7889", + "x7894", + "x7899", + "x7994", +) +#: "For whose education was this loan taken out?" +SCF_EDUCATION_LOAN_WHOSE_COLUMNS: tuple[str, ...] = ( + "x7978", + "x7883", + "x7888", + "x7893", + "x7898", + "x7993", +) +#: "In what year was this loan (or the most recent underlying loan) borrowed?" +SCF_EDUCATION_LOAN_YEAR_COLUMNS: tuple[str, ...] = ( + "x9332", + "x9333", + "x9334", + "x9335", + "x9336", + "x9337", +) +#: "What type of educational program did this loan help finance?" +SCF_EDUCATION_LOAN_PROGRAM_COLUMNS: tuple[str, ...] = ( + "x9338", + "x9339", + "x9340", + "x9341", + "x9342", + "x9343", +) +#: "How much was borrowed for this loan?" (or the balance after the most +#: recent consolidation or refinancing). +SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS: tuple[str, ...] = ( + "x7805", + "x7828", + "x7851", + "x7905", + "x7928", + "x7951", +) +_SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN = "x19" +#: "Original respondent and spouse/partner variables reversed": 1 = reversed. +SCF_REVERSED_COLUMN = "x8000" +_SCF_REVERSED = 1 +#: Loans (1-6) whose whose-education code the 2025 public file did not reverse +#: with the rest of a reversed record: loans 4 and 6 (X7893, X7993). See the +#: module docstring for the evidence. +SCF_WHOSE_EDUCATION_UNREVERSED_LOANS: tuple[int, ...] = (4, 6) +#: Household listing, persons 3-12: relationship to the respondent and age. +SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS: tuple[str, ...] = ( + "x108", + "x114", + "x120", + "x126", + "x132", + "x202", + "x208", + "x214", + "x220", + "x226", +) +SCF_HOUSEHOLD_LISTING_AGE_COLUMNS: tuple[str, ...] = ( + "x110", + "x116", + "x122", + "x128", + "x134", + "x204", + "x210", + "x216", + "x222", + "x228", +) +#: Listing relationship code 4: child (or child-in-law) of R or spouse/partner. +_SCF_LISTING_CHILD = 4 +#: The public file codes listed members' ages in bands (codebook X110): 1-3 as +#: 3, 4-6 as 6, 7-12 as 12, 13-17 as 17, 18-25 as 25, and older ages rounded +#: to the nearest 5 (so 26-27 also read 25), top-coded at 95. Code -> the +#: youngest and oldest age it can stand for. +_SCF_LISTING_AGE_BANDS: Mapping[int, tuple[float, float]] = { + 3: (0.0, 3.0), + 6: (4.0, 6.0), + 12: (7.0, 12.0), + 17: (13.0, 17.0), + 25: (18.0, 27.0), + 95: (93.0, 120.0), +} +#: The first listing age code that is certainly an adult (18-27). +_SCF_LISTING_FIRST_ADULT_CODE = 25 +#: The survey year that ages and years borrowed are counted from. +_SCF_SURVEY_YEAR = 2025 +#: Youngest age at which a loan can have financed a listed child's schooling. +_EARLIEST_STUDENT_AGE = 16.0 +#: A student 24 or older by December 31 of the award year is independent +#: (20 U.S.C. 1087vv(d)(1)(A)), so a parent cannot borrow Parent PLUS for them. +_INDEPENDENT_STUDENT_AGE = 24.0 +#: Aggregate limit on a dependent undergraduate's own Direct Subsidized and +#: Unsubsidized Loans (34 CFR 685.203(e)(1)). +_DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT = 31_000.0 + +_FEDERAL_LOAN_YES = 1 +#: Programs counted as graduate or professional: master's (13), professional +#: school (14) and doctorate (15). +_GRADUATE_PROGRAM_CODES = frozenset({13, 14, 15}) +#: First year counted as a new IBR borrower (20 U.S.C. 1098e(e): no +#: outstanding balance on July 1, 2014). The SCF records only the year. +_IBR_NEW_BORROWER_FIRST_YEAR = 2014 + +_HEAD = "head" +_SPOUSE = "spouse" +_CHILD = "child" +_PARENT_PLUS = "parent_plus" +_CHILD_EDUCATION = "child_education" +#: Whose-education code -> who holds the loan in the build. A loan for a +#: child's education is a co-resident child's own loan or the head's Parent +#: PLUS loan, by the rule in the module docstring. +SCF_WHOSE_EDUCATION_HOLDER: Mapping[int, str] = { + 1: _HEAD, + 2: _SPOUSE, + 3: _CHILD_EDUCATION, + 6: _HEAD, + 7: _HEAD, + -7: _HEAD, +} + +#: Household-total target of the first QRF. +SCF_EDUCATION_DEBT_TOTAL_TARGET = "federal_education_debt" +#: The second QRF's extra predictor: children of the reference person or +#: partner aged 18 or over in the household (SCF listing code 4). +SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR = "adult_children_in_household" +#: Each holder's share of the household total, drawn first in this order and +#: then normalized to sum to one. +SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS: tuple[str, ...] = ( + "parent_plus_share", + "child_own_share", + "head_own_share", + "spouse_own_share", +) +#: Chained targets of the second (borrower-family) QRF. +SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: tuple[str, ...] = ( + *SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS, + "head_graduate_share", + "spouse_graduate_share", + "child_graduate_share", + "head_is_ibr_new_borrower", + "spouse_is_ibr_new_borrower", + "child_is_ibr_new_borrower", +) +#: Every predictor both sides build: the shared SCF predictors plus income +#: excluding capital gains. The diagnostics and ablations compare all of them. +#: Educational attainment joins once microcosm#297 carries it on the CPS side. +SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS: tuple[str, ...] = ( + *SCF_WEALTH_PREDICTORS, + "household_income", +) +#: Candidate predictors the QRFs leave out, each with its reason. Mapping each +#: to its SCF distribution at equal rank within five-year reference-person age +#: bins moves the host's borrower share from 14.2% to 15.8% (interest and +#: dividends) and 14.7% (Social Security and pensions); all candidates together +#: reach 17.3%, the SCF hold-out's level +#: (experiments/receipts/1073-predictor-shift-diagnosis.json). +SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS: Mapping[str, str] = { + "interest_dividend_income": ( + "CPS interest and dividends are tax-record imputations: 42% of host " + "households at reference ages 25-34 have some, against 16% of SCF " + "families." + ), + "social_security_pension_income": ( + "CPS Social Security and pension income is likewise imputed: 18% of " + "host households at reference ages 25-34 have some, against 6% of SCF " + "families." + ), +} +#: The predictors both QRFs condition on. +SCF_EDUCATION_DEBT_PREDICTORS: tuple[str, ...] = tuple( + predictor + for predictor in SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS + if predictor not in SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS +) + + +def scf_education_debt_composition_predictors( + predictors: Sequence[str] = SCF_EDUCATION_DEBT_PREDICTORS, +) -> tuple[str, ...]: + """The second QRF's predictors: the first's, adult children and the total.""" + + return ( + *predictors, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + ) + + +US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS: tuple[str, ...] = ( + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", +) +US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS: tuple[str, ...] = ( + *US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS, + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share", +) + +#: Federal Student Aid portfolio bands (dollars, lower bound inclusive). +SCF_EDUCATION_DEBT_BALANCE_BANDS: tuple[tuple[str, float, float], ...] = ( + ("under_5k", 0.0, 5_000.0), + ("5k_to_10k", 5_000.0, 10_000.0), + ("10k_to_20k", 10_000.0, 20_000.0), + ("20k_to_40k", 20_000.0, 40_000.0), + ("40k_to_60k", 40_000.0, 60_000.0), + ("60k_to_80k", 60_000.0, 80_000.0), + ("80k_to_100k", 80_000.0, 100_000.0), + ("100k_to_200k", 100_000.0, 200_000.0), + ("200k_plus", 200_000.0, np.inf), +) +#: Federal Student Aid borrower age bands (years, inclusive bounds). +SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS: tuple[tuple[str, float, float], ...] = ( + ("24_and_younger", -np.inf, 24.0), + ("25_to_34", 25.0, 34.0), + ("35_to_49", 35.0, 49.0), + ("50_to_61", 50.0, 61.0), + ("62_and_older", 62.0, np.inf), +) +#: SCF ``racecl4`` labels (2025 folds Middle Eastern or North African into 1). +_RACECL4_LABELS: Mapping[int, str] = { + 1: "white_non_hispanic", + 2: "black_non_hispanic", + 3: "hispanic", + 4: "other_or_multiple", +} + +#: Summary-extract columns read beyond the shared predictor surface. +_SUMMARY_EXTRA_COLUMNS: tuple[str, ...] = ( + "y1", + "yy1", + "income", + "kginc", + "racecl4", + "edn_inst", + "educ", +) +#: Person income inputs summed into the recipient household's income, the CPS +#: counterpart of SCF ``income`` less ``kginc``. Absent columns count as zero. +_RECIPIENT_HOUSEHOLD_INCOME_COLUMNS: tuple[str, ...] = ( + "employment_income_before_lsr", + "self_employment_income_before_lsr", + "farm_operations_income", + "farm_rent_income", + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", + "rental_income", + "estate_income", + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents", + "taxable_private_pension_income", + "tax_exempt_private_pension_income", + "taxable_ira_distributions", + "taxable_401k_distributions", + "taxable_403b_distributions", + "taxable_sep_distributions", + "unemployment_compensation", + "alimony_income", + "child_support_received", + "veterans_benefits", + "workers_compensation", + "disability_benefits", +) +_RECIPIENT_INTEREST_DIVIDEND_COLUMNS = ( + "taxable_interest_income", + "tax_exempt_interest_income", + "qualified_dividend_income", + "non_qualified_dividend_income", +) +_RECIPIENT_SS_PENSION_COLUMNS = ( + "social_security_retirement", + "social_security_disability", + "social_security_survivors", + "social_security_dependents", + "taxable_private_pension_income", + "tax_exempt_private_pension_income", +) +_REQUIRED_RECIPIENT_PERSON_COLUMNS: tuple[str, ...] = ( + _HOUSEHOLD_ID_COLUMN, + "age", + "is_female", + "PRDTRACE", + "PRDTHSP", + "A_MARITL", + "A_LINENO", + "A_SPOUSE", + "PH_SEQ", + "PEPAR1", + "PEPAR2", + "employment_income_before_lsr", +) +_OPTIONAL_PARTNER_COLUMN = "PECOHAB" + +#: Age from which a CPS child of the reference person or partner is an adult. +_ADULT_AGE = 18.0 + +#: Weighted share of households holding federal education debt allowed by the +#: signal gate. The SCF 2025 share holding any education debt is 19.9%. +_BORROWER_HOUSEHOLD_SHARE_BAND = (0.08, 0.35) + + +# --------------------------------------------------------------------------- # +# Source files # +# --------------------------------------------------------------------------- # +def _sha256_hexdigest(payload: bytes) -> str: + import hashlib + + return hashlib.sha256(payload).hexdigest() + + +def fetch_scf_public_file( + public_file: ScfPublicFile, + cache_dir: str | Path | None = None, +) -> Path: + """Download, verify and cache one pinned SCF public Stata member. + + A cached member whose SHA-256 matches the pin is reused without a network + call; anything else is fetched from the Federal Reserve, and both the zip + and the member must match their pins before the member is cached. + """ + + import io + import urllib.request + import zipfile + + root = ( + Path(cache_dir).expanduser() + if cache_dir is not None + else Path.home() / ".cache" / "microcosm" / "scf" + ) + root.mkdir(parents=True, exist_ok=True) + target = root / public_file.member + if target.exists() and target.stat().st_size > 0: + if _sha256_hexdigest(target.read_bytes()) == public_file.member_sha256: + return target + + request = urllib.request.Request( + public_file.url, + headers={"User-Agent": _SCF_HTTP_USER_AGENT}, + ) + with urllib.request.urlopen(request, timeout=300) as response: # noqa: S310 + payload = response.read() + digest = _sha256_hexdigest(payload) + if digest != public_file.zip_sha256: + raise ValueError( + f"SCF {public_file.zip_name} failed sha-256 verification: " + f"expected {public_file.zip_sha256}, got {digest}." + ) + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + if public_file.member not in archive.namelist(): + raise ValueError( + f"SCF {public_file.zip_name} is missing member {public_file.member!r}." + ) + member_bytes = archive.read(public_file.member) + digest = _sha256_hexdigest(member_bytes) + if digest != public_file.member_sha256: + raise ValueError( + f"SCF {public_file.member} failed sha-256 verification: " + f"expected {public_file.member_sha256}, got {digest}." + ) + target.write_bytes(member_bytes) + return target + + +def _read_stata_columns( + path: str | Path, + columns: Sequence[str], + *, + label: str, + chunksize: int = 4_000, +) -> pd.DataFrame: + """Read named columns of a Stata file in row chunks. + + The full public file is about 220 MB with over 5,000 variables; reading it + whole would hold every variable in memory. Chunks keep only the requested + columns resident. + """ + + wanted = list(dict.fromkeys(columns)) + try: + with pd.read_stata( + path, + convert_categoricals=False, + columns=wanted, + chunksize=chunksize, + ) as reader: + parts = [chunk for chunk in reader] + except ValueError as error: + if "not found in the Stata data set" not in str(error): + raise + raise ValueError(f"SCF {label} missing required column(s): {error}.") from error + table = pd.concat(parts, ignore_index=True) if parts else pd.DataFrame() + missing = sorted(set(wanted) - set(table.columns)) + if missing: + raise ValueError(f"SCF {label} missing required column(s): {missing}.") + return table.loc[:, wanted] + + +# --------------------------------------------------------------------------- # +# Donor targets # +# --------------------------------------------------------------------------- # +def _numeric(table: pd.DataFrame, column: str) -> np.ndarray: + return ( + pd.to_numeric(table[column], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + + +def _listing_age_range(code: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """The youngest and oldest age each public-file listing age code stands for. + + Codes 30-90 are ages rounded to the nearest 5, so ``code - 2`` to + ``code + 2``; zero or negative (no such person) gives NaN. + """ + + code = np.asarray(code, dtype=np.float64) + youngest = code - 2.0 + oldest = code + 2.0 + for band_code, (low, high) in _SCF_LISTING_AGE_BANDS.items(): + youngest = np.where(code == band_code, low, youngest) + oldest = np.where(code == band_code, high, oldest) + missing = code <= 0 + return np.where(missing, np.nan, youngest), np.where(missing, np.nan, oldest) + + +def _scf_whose_education_codes(full: pd.DataFrame) -> np.ndarray: + """Whose-education codes of loans 1-6, with the X8000 correction applied. + + In a reversed family (X8000 = 1) the public file swaps every respondent and + spouse/partner code so that code 1 is the reference person, except on + loans 4 and 6, whose codes still name the original respondent: those two + codes are swapped here. + """ + + whose = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_WHOSE_COLUMNS] + ).reshape(len(full), len(SCF_EDUCATION_LOAN_WHOSE_COLUMNS)) + reversed_family = _numeric(full, SCF_REVERSED_COLUMN) == _SCF_REVERSED + for loan in SCF_WHOSE_EDUCATION_UNREVERSED_LOANS: + column = whose[:, loan - 1] + whose[:, loan - 1] = np.where( + reversed_family & (column == 1), + 2.0, + np.where(reversed_family & (column == 2), 1.0, column), + ) + return whose + + +def _scf_adult_children(full: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]: + """Listed children aged 18 or over: a (rows x 10) mask and the age codes.""" + + shape = (len(full), len(SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS)) + relationship = np.column_stack( + [_numeric(full, c) for c in SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS] + ).reshape(shape) + age_code = np.column_stack( + [_numeric(full, c) for c in SCF_HOUSEHOLD_LISTING_AGE_COLUMNS] + ).reshape(shape) + adult_child = (relationship == _SCF_LISTING_CHILD) & ( + age_code >= _SCF_LISTING_FIRST_ADULT_CODE + ) + return adult_child, age_code + + +def derive_scf_education_debt_targets(full: pd.DataFrame) -> pd.DataFrame: + """Derive household federal education-debt targets from per-loan items. + + Pure and row-preserving: one output row per input (family, implicate) row. + See the module docstring for the federal, X7179, X8000, whose-education, + co-resident child, IBR and graduate-share rules this applies. + + Returns columns: + ``federal_education_debt``: federal balance after the X7179 rule. + ``head_federal_student_loan_balance``, + ``spouse_federal_student_loan_balance``, + ``child_federal_student_loan_balance``, + ``federal_parent_plus_loan_balance``: its split by holder. + ``head_own_share``, ``spouse_own_share``, ``child_own_share``, + ``parent_plus_share``: each part over the total (zero for a family + without federal education debt). + ``head_is_ibr_new_borrower``, ``spouse_is_ibr_new_borrower``, + ``child_is_ibr_new_borrower``: 0/1. + ``head_graduate_share``, ``spouse_graduate_share``, + ``child_graduate_share``: in [0, 1]. + ``adult_children_in_household``: listed children aged 18 or over. + ``youngest_adult_child_age_code``: the youngest one's listing age code + (0 when there is none). + ``child_borrower_age_code``: the listing age code of the youngest + child a child-own loan fits (0 when there is none). + ``education_debt``: every education loan, federal or not (EDN_INST + without the education-coded installment loans). + ``federal_status_unrecorded_balance``: loans 1-6 balances whose federal + flag is blank (counted as non-federal). + ``remaining_loans_without_detail``: X7179 in a family with no balance + on loans 1-6 (dropped). + """ + + required = ( + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + *SCF_EDUCATION_LOAN_YEAR_COLUMNS, + *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_REVERSED_COLUMN, + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + ) + missing = [column for column in required if column not in full.columns] + if missing: + raise ValueError(f"SCF education-debt derivation missing column(s): {missing}.") + + n = len(full) + balances = np.column_stack( + [np.maximum(_numeric(full, c), 0.0) for c in SCF_EDUCATION_LOAN_BALANCE_COLUMNS] + ).reshape(n, len(SCF_EDUCATION_LOAN_BALANCE_COLUMNS)) + remaining = np.maximum( + _numeric(full, SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN), 0.0 + ) + federal_code = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_FEDERAL_COLUMNS] + ).reshape(balances.shape) + whose = _scf_whose_education_codes(full) + year = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_YEAR_COLUMNS] + ).reshape(balances.shape) + program = np.column_stack( + [_numeric(full, c) for c in SCF_EDUCATION_LOAN_PROGRAM_COLUMNS] + ).reshape(balances.shape) + borrowed = np.column_stack( + [ + np.maximum(_numeric(full, c), 0.0) + for c in SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS + ] + ).reshape(balances.shape) + + has_balance = balances > 0 + federal = has_balance & (federal_code == _FEDERAL_LOAN_YES) + loans_1_6 = balances.sum(axis=1) + gross_up = np.ones(n, dtype=np.float64) + positive_detail = loans_1_6 > 0 + gross_up[positive_detail] = ( + 1.0 + remaining[positive_detail] / loans_1_6[positive_detail] + ) + federal_balance = np.where(federal, balances, 0.0) * gross_up[:, None] + + holder = np.full(balances.shape, _HEAD, dtype=object) + for code, role in SCF_WHOSE_EDUCATION_HOLDER.items(): + holder[whose == code] = role + graduate = np.isin(program, list(_GRADUATE_PROGRAM_CODES)) + + # A loan for a child's education is a co-resident child's own loan when a + # listed child aged 18 or over could have been in school when it was + # borrowed, unless only a parent could have borrowed it: an undergraduate + # amount above a dependent student's own aggregate limit, borrowed while + # the child could still have been dependent. Ages are the public file's + # bands; the arrays are (families x 10 listed persons x 6 loans). + adult_child, age_code = _scf_adult_children(full) + youngest_age, oldest_age = _listing_age_range(age_code) + years_ago = (_SCF_SURVEY_YEAR - year)[:, None, :] + # NaN ages (no such person) compare false, and adult_child excludes them. + fits = adult_child[:, :, None] & ( + oldest_age[:, :, None] - years_ago >= _EARLIEST_STUDENT_AGE + ) + could_be_dependent = ( + fits & (youngest_age[:, :, None] - years_ago < _INDEPENDENT_STUDENT_AGE) + ).any(axis=1) + parents_only = ( + could_be_dependent + & ~graduate + & (borrowed > _DEPENDENT_UNDERGRADUATE_AGGREGATE_LIMIT) + ) + child_education = holder == _CHILD_EDUCATION + childs_own = child_education & fits.any(axis=1) & ~parents_only + holder[child_education] = _PARENT_PLUS + holder[childs_own] = _CHILD + + held = { + role: federal & (holder == role) + for role in (_HEAD, _SPOUSE, _CHILD, _PARENT_PLUS) + } + amount = { + role: np.where(mask, federal_balance, 0.0).sum(axis=1) + for role, mask in held.items() + } + total = amount[_HEAD] + amount[_SPOUSE] + amount[_CHILD] + amount[_PARENT_PLUS] + + def _share(numerator: np.ndarray) -> np.ndarray: + return np.divide( + numerator, + total, + out=np.zeros(n, dtype=np.float64), + where=total > 0, + ) + + new_loan = year >= _IBR_NEW_BORROWER_FIRST_YEAR + + def _new_borrower(holds: np.ndarray) -> np.ndarray: + return (holds.any(axis=1) & np.all(new_loan | ~holds, axis=1)).astype( + np.float64 + ) + + weight = np.where(borrowed > 0, borrowed, balances) + + def _graduate_share(holds: np.ndarray) -> np.ndarray: + denominator = np.where(holds, weight, 0.0).sum(axis=1) + numerator = np.where(holds & graduate, weight, 0.0).sum(axis=1) + return np.divide( + numerator, + denominator, + out=np.zeros(n, dtype=np.float64), + where=denominator > 0, + ) + + youngest_adult = np.where(adult_child, age_code, np.inf).min(axis=1) + fitted_codes = np.where(fits, age_code[:, :, None], np.inf).min(axis=1) + child_borrower = np.where(held[_CHILD], fitted_codes, np.inf).min(axis=1) + + result = pd.DataFrame(index=full.index) + result[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total + result["head_federal_student_loan_balance"] = amount[_HEAD] + result["spouse_federal_student_loan_balance"] = amount[_SPOUSE] + result["child_federal_student_loan_balance"] = amount[_CHILD] + result["federal_parent_plus_loan_balance"] = amount[_PARENT_PLUS] + result["parent_plus_share"] = _share(amount[_PARENT_PLUS]) + result["child_own_share"] = _share(amount[_CHILD]) + result["head_own_share"] = _share(amount[_HEAD]) + result["spouse_own_share"] = _share(amount[_SPOUSE]) + result["head_graduate_share"] = _graduate_share(held[_HEAD]) + result["spouse_graduate_share"] = _graduate_share(held[_SPOUSE]) + result["child_graduate_share"] = _graduate_share(held[_CHILD]) + # The head's flag covers Parent PLUS loans too: 34 CFR 685.209(b)(13)(ii) + # looks at any outstanding Direct or FFEL balance. + result["head_is_ibr_new_borrower"] = _new_borrower(held[_HEAD] | held[_PARENT_PLUS]) + result["spouse_is_ibr_new_borrower"] = _new_borrower(held[_SPOUSE]) + result["child_is_ibr_new_borrower"] = _new_borrower(held[_CHILD]) + result[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = adult_child.sum( + axis=1 + ).astype(np.float64) + result["youngest_adult_child_age_code"] = np.where( + np.isfinite(youngest_adult), youngest_adult, 0.0 + ) + result["child_borrower_age_code"] = np.where( + np.isfinite(child_borrower), child_borrower, 0.0 + ) + result["education_debt"] = loans_1_6 + remaining + result["federal_status_unrecorded_balance"] = np.where( + has_balance & (federal_code == 0), balances, 0.0 + ).sum(axis=1) + result["remaining_loans_without_detail"] = np.where(positive_detail, 0.0, remaining) + return result + + +def load_scf_2025_education_debt_donor( + summary_extract_path: str | Path, + full_extract_path: str | Path, + *, + expected_rows: int | None = SCF_2025_FULL_EXTRACT.rows, +) -> pd.DataFrame: + """Join the 2025 summary and full files and derive the donor table. + + The two files must join one-to-one on ``(y1, yy1)`` across all five + implicates, each must have ``expected_rows`` rows (``None`` skips that + check, for test fixtures), and the per-loan balances must reproduce the + summary extract's ``edn_inst``. Returns one row per (family, implicate) + with a positive weight: the predictors, the targets of + :func:`derive_scf_education_debt_targets` (which include the second QRF's + ``adult_children_in_household``), the ``scf_weight`` donor weight and the + fit-table covariates (``racecl4``, ``income``, ``educ``, ``spouse_age``, + ``yy1``, ``implicate``). + """ + + summary_columns = ( + *dict.fromkeys( + ( + *_SUMMARY_EXTRA_COLUMNS, + "wgt", + "age", + "hhsex", + "racecl5", + "married", + "kids", + "wageinc", + "intdivinc", + "ssretinc", + ) + ), + ) + full_columns = ( + "y1", + "yy1", + _SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN, + *SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + *SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + *SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + *SCF_EDUCATION_LOAN_YEAR_COLUMNS, + *SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + *SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_REVERSED_COLUMN, + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + ) + summary = _read_stata_columns( + summary_extract_path, summary_columns, label="2025 summary extract" + ) + full = _read_stata_columns(full_extract_path, full_columns, label="2025 full file") + keys = ["y1", "yy1"] + for label, table in (("summary", summary), ("full", full)): + if expected_rows is not None and len(table) != int(expected_rows): + raise ValueError( + f"SCF 2025 {label} file has {len(table)} rows; expected " + f"{int(expected_rows)}." + ) + if table.duplicated(keys).any(): + raise ValueError( + f"SCF 2025 {label} file does not have one-to-one (y1, yy1) keys." + ) + keyed_summary = summary.reset_index(drop=True) + keyed_summary["_summary_order"] = np.arange(len(keyed_summary)) + joined = keyed_summary.merge( + full, + on=keys, + how="outer", + validate="one_to_one", + indicator=True, + sort=False, + ) + unmatched = int((joined["_merge"] != "both").sum()) + if unmatched: + raise ValueError( + "SCF 2025 summary/full files have unmatched (y1, yy1) implicates; " + f"a one-to-one join is required ({unmatched} unmatched row(s))." + ) + joined = joined.sort_values("_summary_order").reset_index(drop=True) + implicate = pd.to_numeric(joined["y1"], errors="coerce").astype("int64") % 10 + if sorted(implicate.unique().tolist()) != [1, 2, 3, 4, 5]: + raise ValueError("SCF 2025 rows must carry implicates 1-5 (y1 mod 10).") + + targets = derive_scf_education_debt_targets(joined) + edn_inst = _numeric(joined, "edn_inst") + mismatch = np.abs(edn_inst - targets["education_debt"].to_numpy()) > 1.0 + if mismatch.any(): + raise ValueError( + "SCF 2025 education-loan balances do not reproduce edn_inst on " + f"{int(mismatch.sum())} row(s); the per-loan definition has drifted." + ) + + predictors = _scf_summary_predictor_table(joined) + donor = predictors.reset_index(drop=True) + donor["household_income"] = _numeric(joined, "income") - _numeric(joined, "kginc") + for column in targets.columns: + donor[column] = targets[column].to_numpy(dtype=np.float64) + donor["racecl4"] = _numeric(joined, "racecl4") + donor["income"] = _numeric(joined, "income") + donor["educ"] = _numeric(joined, "educ") + donor["spouse_age"] = _replace_sentinels( + joined[_SCF_FULL_EXTRACT_SPOUSE_AGE_COLUMN] + ).to_numpy(dtype=np.float64) + donor["yy1"] = pd.to_numeric(joined["yy1"], errors="coerce").to_numpy() + donor["implicate"] = implicate.to_numpy() + return donor.loc[donor[_DONOR_WEIGHT_COLUMN] > 0].reset_index(drop=True) + + +# --------------------------------------------------------------------------- # +# Recipients # +# --------------------------------------------------------------------------- # +_REFERENCE_PERSON_RULES = frozenset({"householder_couple", "oldest_adult"}) +_PARTNERED_RULES = frozenset({"spouse_or_partner", "married_spouse"}) +_CHILDREN_RULES = frozenset({"of_reference_or_partner", "of_reference"}) +_INCOME_UNITS = frozenset({"household", "spm_unit", "reference_couple_and_children"}) + + +@dataclass(frozen=True) +class RecipientDefinition: + """How the CPS side builds the SCF predictors for one household. + + Each field names the CPS construction of one SCF concept: + + - ``reference_person``: ``householder_couple`` takes the CPS householder + (lowest line number) and, when a spouse or cohabiting partner is present, + the man of a mixed-sex couple or else the older partner, which is how + the SCF orders its economically dominant couple. ``oldest_adult`` is the + auto-loan rule (the only adult, the man of a two-adult married couple, + else the oldest adult). + - ``partnered``: ``spouse_or_partner`` is SCF ``married`` (married or living + with a partner, X8023 in 1-2): a spouse (``A_SPOUSE``) or cohabiting + partner (``PECOHAB``) is present. ``married_spouse`` is ``A_MARITL`` 1-2. + - ``children``: ``of_reference_or_partner`` is SCF ``kids`` (children, + step-children and children-in-law of the reference person or partner, + any age, anywhere in the household); ``of_reference`` counts only the + reference person's own children through ``PEPAR1``/``PEPAR2``. + - ``income_unit``: the people whose incomes are summed. ``household`` sums + every member; ``spm_unit`` the reference person's SPM unit; + ``reference_couple_and_children`` the reference person, spouse or + partner and their children. + """ + + reference_person: str = "householder_couple" + partnered: str = "spouse_or_partner" + children: str = "of_reference_or_partner" + income_unit: str = "reference_couple_and_children" + + def __post_init__(self) -> None: + for value, allowed, name in ( + (self.reference_person, _REFERENCE_PERSON_RULES, "reference_person"), + (self.partnered, _PARTNERED_RULES, "partnered"), + (self.children, _CHILDREN_RULES, "children"), + (self.income_unit, _INCOME_UNITS, "income_unit"), + ): + if value not in allowed: + raise ValueError( + f"RecipientDefinition.{name} must be one of {sorted(allowed)}." + ) + + +#: The definition the stage uses (see the module docstring for the evidence). +SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION = RecipientDefinition() +#: The definition of the first #1073 receipt, kept for its before/after table. +SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION = RecipientDefinition( + reference_person="oldest_adult", + partnered="married_spouse", + children="of_reference", + income_unit="household", +) + + +@dataclass(frozen=True) +class _Recipients: + """Household-ordered predictors and each household's recipient rows.""" + + features: pd.DataFrame + head_position: np.ndarray + spouse_position: np.ndarray # -1 where no spouse or partner is present + child_position: np.ndarray # the youngest adult child; -1 where none + + +def _line_lookup(person: pd.DataFrame) -> dict[tuple[object, int], int]: + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + return { + (household_id, int(line_number)): position + for position, (household_id, line_number) in enumerate( + zip(household, line, strict=True) + ) + } + + +def _resolve_partner_positions( + person: pd.DataFrame, head_positions: np.ndarray +) -> np.ndarray: + """Positional row of each head's spouse (A_SPOUSE) or partner (PECOHAB).""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + lookup = _line_lookup(person) + pointers = [pd.to_numeric(person["A_SPOUSE"], errors="coerce").fillna(0).to_numpy()] + if _OPTIONAL_PARTNER_COLUMN in person.columns: + pointers.append( + pd.to_numeric(person[_OPTIONAL_PARTNER_COLUMN], errors="coerce") + .fillna(0) + .to_numpy() + ) + result = np.full(len(head_positions), -1, dtype=np.int64) + for index, head in enumerate(head_positions): + for pointer in pointers: + target = int(pointer[head]) + if target <= 0: + continue + partner = lookup.get((household[head], target)) + if partner is not None and partner != head: + result[index] = partner + break + return result + + +def _householder_couple_positions( + person: pd.DataFrame, order: np.ndarray +) -> np.ndarray: + """The SCF-ordered reference person of each household, aligned to ``order``.""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(9_999).to_numpy() + position = np.arange(len(person)) + ranked = np.lexsort((position, line, household)) + first = np.ones(len(person), dtype=bool) + first[1:] = household[ranked][1:] != household[ranked][:-1] + householders = ranked[first] + by_household = pd.Series(householders, index=household[householders]).reindex(order) + if by_household.isna().any(): + missing_ids = order[by_household.isna().to_numpy()][:5].tolist() + raise ValueError( + "SCF education-debt receiver household alignment failed: " + f"no householder for household id(s) {missing_ids}." + ) + householder = by_household.to_numpy(dtype=np.int64) + partner = _resolve_partner_positions(person, householder) + female = person["is_female"].astype(bool).to_numpy() + age = pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + has_partner = partner >= 0 + safe_partner = np.where(has_partner, partner, householder) + mixed = female[householder] != female[safe_partner] + partner_first = np.where( + mixed, + female[householder], # the man of a mixed-sex couple + age[safe_partner] > age[householder], # else the older partner + ) + return np.where(has_partner & partner_first, safe_partner, householder) + + +def _children_counts( + person: pd.DataFrame, + head_position: np.ndarray, + spouse_position: np.ndarray, + *, + include_partner: bool, +) -> np.ndarray: + """Household members whose CPS parent pointer names the head (or partner).""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + parent_of: dict[tuple[object, int], int] = { + (household[head], int(line[head])): index + for index, head in enumerate(head_position) + } + if include_partner: + for index, spouse in enumerate(spouse_position): + if spouse >= 0: + parent_of.setdefault((household[spouse], int(line[spouse])), index) + counts = np.zeros(len(head_position), dtype=np.float64) + pointers = [ + pd.to_numeric(person[column], errors="coerce").fillna(-1).to_numpy() + for column in ("PEPAR1", "PEPAR2") + ] + for row in range(len(person)): + owners = { + parent_of.get((household[row], int(pointer[row]))) + for pointer in pointers + if pointer[row] > 0 + } + owners.discard(None) + # A child of both partners counts once. + if owners: + counts[next(iter(owners))] += 1.0 + return counts + + +def _adult_children( + person: pd.DataFrame, + head_position: np.ndarray, + spouse_position: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Children of the head or partner aged 18 or over, and the youngest one. + + A child is a member whose CPS parent pointer (``PEPAR1``/``PEPAR2``) names + the head or the spouse or partner, as SCF listing code 4 counts children of + the respondent or spouse/partner. Returns the count per household and the + positional row of the youngest (lowest line number on a tie), or -1. + """ + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + age = pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + parent_of: dict[tuple[object, int], int] = { + (household[head], int(line[head])): index + for index, head in enumerate(head_position) + } + for index, spouse in enumerate(spouse_position): + if spouse >= 0: + parent_of.setdefault((household[spouse], int(line[spouse])), index) + pointers = [ + pd.to_numeric(person[column], errors="coerce").fillna(-1).to_numpy() + for column in ("PEPAR1", "PEPAR2") + ] + counts = np.zeros(len(head_position), dtype=np.float64) + youngest = np.full(len(head_position), -1, dtype=np.int64) + for row in np.flatnonzero(age >= _ADULT_AGE): + owners = { + parent_of.get((household[row], int(pointer[row]))) + for pointer in pointers + if pointer[row] > 0 + } + owners.discard(None) + if not owners: + continue + owner = next(iter(owners)) + counts[owner] += 1.0 + current = youngest[owner] + if current < 0 or (age[row], line[row]) < (age[current], line[current]): + youngest[owner] = row + return counts, youngest + + +def _income_unit_owner( + person: pd.DataFrame, + head_position: np.ndarray, + spouse_position: np.ndarray, + unit: str, +) -> np.ndarray: + """For each person, the household row whose income unit includes them, or -1.""" + + household = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + if unit == "household": + index = pd.Series(np.arange(len(head_position)), index=household[head_position]) + return index.reindex(household).to_numpy(dtype=np.int64) + owner = np.full(len(person), -1, dtype=np.int64) + if unit == "spm_unit": + if "person_spm_unit_id" not in person.columns: + raise ValueError("The spm_unit income unit requires person_spm_unit_id.") + spm = person["person_spm_unit_id"].to_numpy() + index = pd.Series(np.arange(len(head_position)), index=spm[head_position]) + index = index[~index.index.duplicated()] + mapped = index.reindex(spm).to_numpy(dtype=np.float64) + owner = np.where(np.isnan(mapped), -1, mapped).astype(np.int64) + owner[head_position] = np.arange(len(head_position)) + has_spouse = spouse_position >= 0 + owner[spouse_position[has_spouse]] = np.flatnonzero(has_spouse) + if unit == "reference_couple_and_children": + line = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(-1).to_numpy() + parent_of = { + (household[row], int(line[row])): owner[row] + for row in np.flatnonzero(owner >= 0) + } + for column in ("PEPAR1", "PEPAR2"): + pointer = ( + pd.to_numeric(person[column], errors="coerce").fillna(-1).to_numpy() + ) + for row in np.flatnonzero((owner < 0) & (pointer > 0)): + found = parent_of.get((household[row], int(pointer[row]))) + if found is not None: + owner[row] = found + return owner + + +def _recipients( + frame: Frame, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, +) -> _Recipients: + person = frame.table("person") + household = frame.table("household") + missing = sorted(set(_REQUIRED_RECIPIENT_PERSON_COLUMNS) - set(person.columns)) + if missing: + raise ValueError( + "US SCF education-debt imputation requires recipient person " + f"column(s): {missing}." + ) + if "household_id" not in household.columns: + raise ValueError("US SCF education-debt imputation requires household_id.") + + order = household["household_id"].to_numpy() + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + if definition.reference_person == "householder_couple": + head_position = _householder_couple_positions(person, order) + else: + mask = _scf_reference_person_mask(person) + head_rows = np.flatnonzero(mask) + head_by_household = pd.Series(head_rows, index=household_ids[head_rows]) + if head_by_household.index.duplicated().any(): + raise ValueError( + "SCF education-debt receiver selected two reference persons." + ) + aligned = head_by_household.reindex(order) + if aligned.isna().any(): + missing_ids = order[aligned.isna().to_numpy()][:5].tolist() + raise ValueError( + "SCF education-debt receiver household alignment failed: " + f"no reference person for household id(s) {missing_ids}." + ) + head_position = aligned.to_numpy(dtype=np.int64) + spouse_position = _resolve_partner_positions(person, head_position) + + features = pd.DataFrame(index=np.arange(len(head_position))) + features["age"] = ( + pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + )[head_position] + features["is_female"] = ( + person["is_female"].astype(bool).astype(float).to_numpy()[head_position] + ) + features["cps_race"] = _recipient_cps_race(person)[head_position] + if definition.partnered == "spouse_or_partner": + features["is_married"] = (spouse_position >= 0).astype(np.float64) + else: + features["is_married"] = ( + pd.to_numeric(person["A_MARITL"], errors="coerce") + .isin([1, 2]) + .astype(float) + .to_numpy()[head_position] + ) + if definition.children == "of_reference_or_partner": + features["own_children_in_household"] = _children_counts( + person, head_position, spouse_position, include_partner=True + ) + else: + features["own_children_in_household"] = _own_children_in_household(person)[ + head_position + ] + + owner = _income_unit_owner( + person, head_position, spouse_position, definition.income_unit + ) + member = owner >= 0 + for column, values in ( + ( + "employment_income", + np.maximum( + pd.to_numeric(person["employment_income_before_lsr"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64), + 0.0, + ), + ), + ( + "interest_dividend_income", + _sum_present(person, _RECIPIENT_INTEREST_DIVIDEND_COLUMNS), + ), + ( + "social_security_pension_income", + _sum_present(person, _RECIPIENT_SS_PENSION_COLUMNS), + ), + ( + "household_income", + _sum_present(person, _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS), + ), + ): + sums = np.zeros(len(head_position), dtype=np.float64) + np.add.at(sums, owner[member], values[member]) + features[column] = sums + adult_children, child_position = _adult_children( + person, head_position, spouse_position + ) + features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = adult_children + features.index = household.index + return _Recipients( + features=features.loc[ + :, + [ + *SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + ], + ], + head_position=head_position, + spouse_position=spouse_position, + child_position=child_position, + ) + + +# --------------------------------------------------------------------------- # +# Imputation # +# --------------------------------------------------------------------------- # +def _stage_seed(seed: int, step: int) -> int: + """A model seed for one of the stage's two QRFs, independent of other stages.""" + + sequence = np.random.SeedSequence([int(seed), 1073, int(step)]) + return int(sequence.generate_state(1, dtype=np.uint32)[0]) + + +def _fit_frame(donor: pd.DataFrame, columns: Sequence[str]) -> pd.DataFrame: + missing = [column for column in columns if column not in donor.columns] + if missing: + raise ValueError(f"SCF education-debt donor missing column(s): {missing}.") + fit = donor.loc[:, list(columns)].copy() + for column in columns: + fit[column] = pd.to_numeric(fit[column], errors="coerce") + if not np.isfinite(fit[column].to_numpy(dtype=np.float64)).all(): + raise ValueError( + f"SCF education-debt donor column {column!r} contains nonfinite values." + ) + return fit + + +def _split_whole_dollars(total: np.ndarray, shares: np.ndarray) -> np.ndarray: + """Split whole-dollar totals by row shares; a zero share stays zero. + + Each part is the floor of its share of the total, and the few dollars of + remainder go to the row's largest share, so the parts sum to the total. + """ + + parts = np.floor(total[:, None] * shares) + remainder = total - parts.sum(axis=1) + largest = np.argmax(shares, axis=1) + parts[np.arange(len(total)), largest] += remainder + return parts + + +def draw_scf_education_debt_households( + features: pd.DataFrame, + donor: pd.DataFrame, + *, + seed: int, + predictors: Sequence[str] = SCF_EDUCATION_DEBT_PREDICTORS, + n_estimators: int = _DEFAULT_N_ESTIMATORS, +) -> pd.DataFrame: + """Draw household totals and their composition for recipient rows. + + ``features`` carries the predictor columns and + ``adult_children_in_household``, one row per recipient household (or + held-out SCF family). Returns, on the same index, the drawn total and the + normalized composition: the head's, spouse's and adult child's own + balances, the head's Parent PLUS balance, the three graduate shares and the + three new-borrower flags. Used by the build and by the SCF hold-out fit + check. + + The four holder shares are drawn in turn (Parent PLUS first, so a family + unlike any Parent PLUS donor draws none) and normalized to sum to one; a + borrower whose every share is drawn zero holds its own loan. A child's + share drawn for a household without an adult child is the head's Parent + PLUS loan: as on the SCF side, a loan for a child who is not in the + household is the parent's. + """ + + from microcosm.fit import QRF + + predictors = list(predictors) + total_fit = _fit_frame( + donor, (*predictors, SCF_EDUCATION_DEBT_TOTAL_TARGET, _DONOR_WEIGHT_COLUMN) + ) + total_model = QRF(n_estimators=int(n_estimators), seed=_stage_seed(seed, 1)).fit( + total_fit, + predictors=predictors, + targets=[SCF_EDUCATION_DEBT_TOTAL_TARGET], + weights=_DONOR_WEIGHT_COLUMN, + ) + drawn_total = np.maximum( + np.asarray( + total_model.predict(features.loc[:, predictors], release_models=True)[ + SCF_EDUCATION_DEBT_TOTAL_TARGET + ], + dtype=np.float64, + ), + 0.0, + ) + drawn_total = np.round(drawn_total) + + result = pd.DataFrame(index=features.index) + result[SCF_EDUCATION_DEBT_TOTAL_TARGET] = drawn_total + for column in SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: + result[column] = 0.0 + borrowers = drawn_total > 0 + if borrowers.any(): + composition_predictors = list( + scf_education_debt_composition_predictors(predictors) + ) + borrower_donor = donor.loc[ + pd.to_numeric(donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], errors="coerce") > 0 + ] + composition_fit = _fit_frame( + borrower_donor, + ( + *composition_predictors, + *SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + _DONOR_WEIGHT_COLUMN, + ), + ) + composition_model = QRF( + n_estimators=int(n_estimators), seed=_stage_seed(seed, 2) + ).fit( + composition_fit, + predictors=composition_predictors, + targets=list(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS), + weights=_DONOR_WEIGHT_COLUMN, + ) + recipients = features.loc[borrowers, composition_predictors[:-1]].copy() + recipients[SCF_EDUCATION_DEBT_TOTAL_TARGET] = drawn_total[borrowers] + drawn = composition_model.predict(recipients, release_models=True) + for column in SCF_EDUCATION_DEBT_COMPOSITION_TARGETS: + result.loc[borrowers, column] = np.asarray(drawn[column], dtype=np.float64) + + shares = np.column_stack( + [ + result[column].clip(0.0, 1.0).to_numpy(dtype=np.float64) + for column in SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS + ] + ) + plus, child, head, spouse = range(len(SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS)) + shares[~borrowers] = 0.0 + shares[borrowers & (shares.sum(axis=1) <= 0), head] = 1.0 + no_adult_child = ( + pd.to_numeric( + features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR], errors="coerce" + ) + .fillna(0.0) + .to_numpy() + <= 0 + ) + child_new = result["child_is_ibr_new_borrower"].to_numpy() >= 0.5 + head_new = result["head_is_ibr_new_borrower"].to_numpy() >= 0.5 + moved = no_adult_child & (shares[:, child] > 0) + # The head then holds Parent PLUS for that child too: new only when every + # loan the head holds is. + head_held = (shares[:, head] + shares[:, plus]) > 0 + head_new = np.where( + moved, np.where(head_held, head_new, True) & child_new, head_new + ) + shares[moved, plus] += shares[moved, child] + shares[moved, child] = 0.0 + sums = shares.sum(axis=1) + shares = np.divide( + shares, sums[:, None], out=np.zeros_like(shares), where=sums[:, None] > 0 + ) + parts = _split_whole_dollars(drawn_total, shares) + result["head_federal_student_loan_balance"] = parts[:, head] + result["spouse_federal_student_loan_balance"] = parts[:, spouse] + result["child_federal_student_loan_balance"] = parts[:, child] + result["federal_parent_plus_loan_balance"] = parts[:, plus] + for prefix, column in (("head", head), ("spouse", spouse), ("child", child)): + graduate = result[f"{prefix}_graduate_share"].clip(0.0, 1.0).to_numpy() + result[f"{prefix}_graduate_share"] = np.where( + parts[:, column] > 0, graduate, 0.0 + ) + result["head_is_ibr_new_borrower"] = ( + ((parts[:, head] + parts[:, plus]) > 0) & head_new + ).astype(np.float64) + result["spouse_is_ibr_new_borrower"] = ( + (parts[:, spouse] > 0) + & (result["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5) + ).astype(np.float64) + result["child_is_ibr_new_borrower"] = ((parts[:, child] > 0) & child_new).astype( + np.float64 + ) + for index, column in enumerate(SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS): + result[column] = shares[:, index] + return result + + +def impute_us_scf_education_debt( + frame: Frame, + donor: pd.DataFrame, + *, + seed: int, + n_estimators: int = _DEFAULT_N_ESTIMATORS, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, + predictors: Sequence[str] = SCF_EDUCATION_DEBT_PREDICTORS, +) -> pd.DataFrame: + """Impute the four person columns onto a US frame. + + Returns a DataFrame indexed like the person table with + :data:`US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS`; only reference persons, + their spouses or partners, and the youngest child of either aged 18 or + over can be nonzero. ``definition`` selects the CPS construction of the + SCF predictors and ``predictors`` the subset the QRFs condition on; the + receipts pass the first receipt's choices for their before/after table. + """ + + recipients = _recipients(frame, definition) + drawn = draw_scf_education_debt_households( + recipients.features, + donor, + seed=seed, + predictors=predictors, + n_estimators=n_estimators, + ) + person = frame.table("person") + n_person = len(person) + own = np.zeros(n_person, dtype=np.float64) + parent_plus = np.zeros(n_person, dtype=np.float64) + graduate = np.zeros(n_person, dtype=np.float64) + new_borrower = np.zeros(n_person, dtype=bool) + + head = recipients.head_position + spouse = recipients.spouse_position + child = recipients.child_position + has_spouse = spouse >= 0 + has_child = child >= 0 + head_own = drawn["head_federal_student_loan_balance"].to_numpy() + spouse_own = drawn["spouse_federal_student_loan_balance"].to_numpy() + child_own = drawn["child_federal_student_loan_balance"].to_numpy() + head_plus = drawn["federal_parent_plus_loan_balance"].to_numpy() + head_graduate = drawn["head_graduate_share"].to_numpy() + spouse_graduate = drawn["spouse_graduate_share"].to_numpy() + child_graduate = drawn["child_graduate_share"].to_numpy() + head_new = drawn["head_is_ibr_new_borrower"].to_numpy() >= 0.5 + spouse_new = drawn["spouse_is_ibr_new_borrower"].to_numpy() >= 0.5 + child_new = drawn["child_is_ibr_new_borrower"].to_numpy() >= 0.5 + # The draw gives a child's balance only to households with an adult child, + # which is exactly where the youngest one was found. + if ((child_own > 0) & ~has_child).any(): + raise ValueError( + "SCF education-debt draw gave a child's balance to a household " + "without an adult child." + ) + + # Without a spouse or partner present, the reference person holds the + # spouse's draw too: a balance-weighted graduate share, and new-borrower + # status only when both holders' loans are new. + moved = ~has_spouse & (spouse_own > 0) + combined_own = np.where(moved, head_own + spouse_own, head_own) + combined_graduate = np.where( + moved, + np.divide( + head_own * head_graduate + spouse_own * spouse_graduate, + combined_own, + out=np.zeros_like(combined_own), + where=combined_own > 0, + ), + head_graduate, + ) + head_holds_any = (head_own + head_plus) > 0 + combined_new = np.where( + moved, + np.where(head_holds_any, head_new, True) & spouse_new, + head_new, + ) + own[head] = combined_own + parent_plus[head] = head_plus + graduate[head] = combined_graduate + new_borrower[head] = combined_new + spouse_rows = spouse[has_spouse] + own[spouse_rows] = spouse_own[has_spouse] + graduate[spouse_rows] = spouse_graduate[has_spouse] + new_borrower[spouse_rows] = spouse_new[has_spouse] + child_rows = child[has_child] + own[child_rows] = child_own[has_child] + graduate[child_rows] = child_graduate[has_child] + new_borrower[child_rows] = child_new[has_child] + + holds_any = (own + parent_plus) > 0 + result = pd.DataFrame(index=person.index) + result["federal_student_loan_balance"] = own + result["federal_parent_plus_loan_balance"] = parent_plus + result["is_income_based_repayment_new_borrower"] = new_borrower & holds_any + result["federal_student_loan_graduate_share"] = np.where(own > 0, graduate, 0.0) + return result + + +def _surface_has_signal(person: pd.DataFrame) -> bool: + if not all(column in person for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS): + return False + for column in US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS: + values = pd.to_numeric(person[column], errors="coerce") + if values.isna().any() or values.nunique() < 2 or not (values > 0).any(): + return False + return True + + +def with_us_scf_education_debt_inputs( + frame: Frame, + *, + seed: int, + time_period: int, + scf_education_debt_donor: pd.DataFrame, + n_estimators: int = _DEFAULT_N_ESTIMATORS, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, +) -> Frame: + """Write the four education-debt person inputs; idempotent on a signal.""" + + del time_period # The draw depends on the seed only; kept for the stage API. + if frame.schema != US_SCHEMA: + raise ValueError("US SCF education-debt inputs require the US schema.") + us_scf_education_debt_stage_spec() + person = frame.table("person") + if _surface_has_signal(person): + return frame + imputed = impute_us_scf_education_debt( + frame, + scf_education_debt_donor, + seed=int(seed), + n_estimators=int(n_estimators), + definition=definition, + ) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS: + tables["person"][column] = imputed[column].to_numpy() + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +# --------------------------------------------------------------------------- # +# Stage declaration, diagnostics and gate # +# --------------------------------------------------------------------------- # +def us_scf_education_debt_stage_spec() -> SourceStageSpec: + """Load the ``scf_wealth`` declaration and check its 2025 education-debt part.""" + + manifest = load_source_manifest( + files("microcosm.build.us").joinpath("source_stages.json") + ) + spec = manifest.stage_map()[_SCF_STAGE_NAME] + missing = sorted(set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) - set(spec.outputs)) + if missing: + raise ValueError( + f"{_SCF_STAGE_NAME!r} manifest stage does not declare education-debt " + f"output(s) {missing}." + ) + for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT): + declared = [ + artifact + for artifact in spec.artifacts + if artifact.get("locator") == public_file.url + ] + if len(declared) != 1: + raise ValueError( + f"{_SCF_STAGE_NAME!r} manifest must declare {public_file.url} once." + ) + artifact = declared[0] + expected = { + "vintage": SCF_EDUCATION_DEBT_VINTAGE, + "member": public_file.member, + "sha256": public_file.zip_sha256, + "member_sha256": public_file.member_sha256, + } + drift = { + key: (artifact.get(key), value) + for key, value in expected.items() + if artifact.get(key) != value + } + if drift: + raise ValueError( + f"{_SCF_STAGE_NAME!r} SCF {SCF_EDUCATION_DEBT_VINTAGE} artifact pin " + f"drift for {public_file.member}: {drift}." + ) + expected_fits = { + "scf_2025_education_debt": ( + SCF_EDUCATION_DEBT_PREDICTORS, + (SCF_EDUCATION_DEBT_TOTAL_TARGET,), + ), + "scf_2025_education_debt_borrowers": ( + scf_education_debt_composition_predictors(), + SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + ), + } + for donor, (predictors, targets) in expected_fits.items(): + fits = [ + operation + for operation in spec.operations + if operation.kind == "fit_weighted_qrf" + and operation.parameters.get("donor") == donor + ] + if ( + len(fits) != 1 + or tuple(fits[0].parameters.get("predictors", ())) != predictors + or tuple(fits[0].parameters.get("targets", ())) != targets + ): + raise ValueError( + f"{_SCF_STAGE_NAME!r} education-debt QRF {donor!r} predictors or " + "targets have drifted." + ) + return spec + + +def _weighted_quantile(values: np.ndarray, weights: np.ndarray, q: float) -> float: + """First value whose cumulative weight reaches ``q`` of the total.""" + + if len(values) == 0 or float(weights.sum()) <= 0: + return float("nan") + order = np.argsort(values, kind="mergesort") + cumulative = np.cumsum(weights[order]) + index = int(np.searchsorted(cumulative, q * cumulative[-1], side="left")) + return float(values[order][min(index, len(values) - 1)]) + + +def _band_shares( + values: np.ndarray, + weights: np.ndarray, + bands: Sequence[tuple[str, float, float]], + *, + inclusive_upper: bool, +) -> dict[str, float]: + total = float(weights.sum()) + shares: dict[str, float] = {} + for label, low, high in bands: + if inclusive_upper: + inside = (values >= low) & (values <= high) + else: + inside = (values >= low) & (values < high) + shares[label] = float(weights[inside].sum()) / total if total > 0 else 0.0 + return shares + + +def _income_quintiles(income: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Weighted income quintile (1-5) of each household.""" + + order = np.argsort(income, kind="mergesort") + cumulative = np.cumsum(weights[order]) + total = cumulative[-1] if len(cumulative) else 0.0 + quintile = np.empty(len(income), dtype=np.int64) + if total <= 0: + quintile[:] = 1 + return quintile + # Midpoint rule: a household's rank is the cumulative weight at its centre. + midpoint = (cumulative - weights[order] / 2.0) / total + quintile[order] = np.minimum((midpoint * 5).astype(np.int64) + 1, 5) + return quintile + + +def education_debt_fit_table( + *, + household_debt: np.ndarray, + household_weight: np.ndarray, + head_age: np.ndarray, + race_group: np.ndarray, + income_quintile: np.ndarray, + borrower_balance: np.ndarray, + borrower_age: np.ndarray, + borrower_weight: np.ndarray, + parent_plus_total: float, + graduate_numerator: float, + graduate_denominator: float, + new_borrower_weight: float, +) -> dict[str, object]: + """The microcosm#1073 fit table from household and borrower arrays. + + Household statistics follow the Fed's SCF tables (share of households with + federal education debt, mean and median among them, by the age of the + reference person, race and income quintile). Borrower statistics follow + Federal Student Aid's portfolio tables: people holding a federal balance + (own plus Parent PLUS), by balance band and borrower age band. + """ + + household_debt = np.asarray(household_debt, dtype=np.float64) + household_weight = np.asarray(household_weight, dtype=np.float64) + borrower = household_debt > 0 + total_weight = float(household_weight.sum()) + + def _group(mask: np.ndarray) -> dict[str, float]: + weight = household_weight[mask] + debt = household_debt[mask] + holders = debt > 0 + group_weight = float(weight.sum()) + holder_weight = float(weight[holders].sum()) + return { + "household_weight": group_weight, + "share_with_federal_education_debt": ( + holder_weight / group_weight if group_weight > 0 else float("nan") + ), + "mean_among_borrowers": ( + float(np.average(debt[holders], weights=weight[holders])) + if holder_weight > 0 + else float("nan") + ), + "median_among_borrowers": _weighted_quantile( + debt[holders], weight[holders], 0.5 + ), + } + + head_age = np.asarray(head_age, dtype=np.float64) + race_group = np.asarray(race_group) + income_quintile = np.asarray(income_quintile) + borrower_balance = np.asarray(borrower_balance, dtype=np.float64) + borrower_age = np.asarray(borrower_age, dtype=np.float64) + borrower_weight = np.asarray(borrower_weight, dtype=np.float64) + federal_total = float(np.dot(household_debt, household_weight)) + return { + "households": { + **_group(np.ones(len(household_debt), dtype=bool)), + "total_household_weight": total_weight, + "federal_education_debt_total": federal_total, + "parent_plus_share_of_total": ( + parent_plus_total / federal_total if federal_total > 0 else float("nan") + ), + }, + "household_balance_bands": _band_shares( + household_debt[borrower], + household_weight[borrower], + SCF_EDUCATION_DEBT_BALANCE_BANDS, + inclusive_upper=False, + ), + "by_head_age_band": { + label: _group((head_age >= low) & (head_age <= high)) + for label, low, high in SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS + }, + "by_race": { + label: _group(race_group == code) for code, label in _RACECL4_LABELS.items() + }, + "by_income_quintile": { + str(quintile): _group(income_quintile == quintile) + for quintile in range(1, 6) + }, + "borrowers": { + "borrower_weight": float(borrower_weight.sum()), + "mean_balance": ( + float(np.average(borrower_balance, weights=borrower_weight)) + if float(borrower_weight.sum()) > 0 + else float("nan") + ), + "median_balance": _weighted_quantile( + borrower_balance, borrower_weight, 0.5 + ), + "balance_bands": _band_shares( + borrower_balance, + borrower_weight, + SCF_EDUCATION_DEBT_BALANCE_BANDS, + inclusive_upper=False, + ), + "age_bands": _band_shares( + np.floor(borrower_age), + borrower_weight, + SCF_EDUCATION_DEBT_BORROWER_AGE_BANDS, + inclusive_upper=True, + ), + "graduate_share_of_own_original_principal": ( + graduate_numerator / graduate_denominator + if graduate_denominator > 0 + else float("nan") + ), + "ibr_new_borrower_share": ( + new_borrower_weight / float(borrower_weight.sum()) + if float(borrower_weight.sum()) > 0 + else float("nan") + ), + }, + } + + +def _spread_listing_ages( + codes: np.ndarray, weights: np.ndarray, balances: np.ndarray +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Spread each listing age code's weight evenly over the ages it stands for. + + The public file bands listed members' ages (18-27 read 25), so a child + borrower's weight is split evenly across the whole years of the band for + the borrower age table. Returns ages, weights and balances, one row per + (borrower, year of age). + """ + + youngest, oldest = _listing_age_range(codes) + ages: list[np.ndarray] = [] + spread: list[np.ndarray] = [] + values: list[np.ndarray] = [] + for low, high, weight, balance in zip( + youngest, oldest, weights, balances, strict=True + ): + years = np.arange(low, min(high, 99.0) + 1.0) + ages.append(years) + spread.append(np.full(len(years), weight / len(years))) + values.append(np.full(len(years), balance)) + if not ages: + return np.zeros(0), np.zeros(0), np.zeros(0) + return np.concatenate(ages), np.concatenate(spread), np.concatenate(values) + + +def scf_education_debt_donor_fit_table( + donor: pd.DataFrame, + *, + imputed: pd.DataFrame | None = None, +) -> dict[str, object]: + """Fit table of the SCF donor itself, or of draws made for its rows. + + ``imputed`` (from :func:`draw_scf_education_debt_households`, on the + donor's index) replaces the observed targets, so a hold-out draw and the + observed SCF are tabulated identically. Implicates are pooled with + ``wgt`` (each implicate carries one fifth of the population), as the Fed + pools them for medians. Borrowers are the reference person, the spouse or + partner and the adult child holding a child-own balance, whose age is the + child's listing age band (the child the loan fits when observed, the + youngest adult child when drawn), spread evenly over the band. + """ + + source = donor if imputed is None else imputed + weight = donor[_DONOR_WEIGHT_COLUMN].to_numpy(dtype=np.float64) + + def _column(name: str) -> np.ndarray: + return source[name].to_numpy(dtype=np.float64) + + head_own = _column("head_federal_student_loan_balance") + spouse_own = _column("spouse_federal_student_loan_balance") + child_own = _column("child_federal_student_loan_balance") + parent_plus = _column("federal_parent_plus_loan_balance") + head_balance = head_own + parent_plus + head_holds = head_balance > 0 + spouse_holds = spouse_own > 0 + child_holds = child_own > 0 + head_age = donor["age"].to_numpy(dtype=np.float64) + spouse_age = donor["spouse_age"].to_numpy(dtype=np.float64) + child_age_code = donor[ + "child_borrower_age_code" + if imputed is None + else "youngest_adult_child_age_code" + ].to_numpy(dtype=np.float64) + child_ages, child_weights, child_balances = _spread_listing_ages( + child_age_code[child_holds], weight[child_holds], child_own[child_holds] + ) + head_graduate = _column("head_graduate_share") + spouse_graduate = _column("spouse_graduate_share") + child_graduate = _column("child_graduate_share") + head_new = _column("head_is_ibr_new_borrower") >= 0.5 + spouse_new = _column("spouse_is_ibr_new_borrower") >= 0.5 + child_new = _column("child_is_ibr_new_borrower") >= 0.5 + income = donor["household_income"].to_numpy(dtype=np.float64) + return education_debt_fit_table( + household_debt=head_own + spouse_own + child_own + parent_plus, + household_weight=weight, + head_age=head_age, + race_group=donor["racecl4"].to_numpy(), + income_quintile=_income_quintiles(income, weight), + borrower_balance=np.concatenate( + [head_balance[head_holds], spouse_own[spouse_holds], child_balances] + ), + borrower_age=np.concatenate( + [head_age[head_holds], spouse_age[spouse_holds], child_ages] + ), + borrower_weight=np.concatenate( + [weight[head_holds], weight[spouse_holds], child_weights] + ), + parent_plus_total=float(np.dot(parent_plus, weight)), + graduate_numerator=float( + np.dot( + head_own * head_graduate + + spouse_own * spouse_graduate + + child_own * child_graduate, + weight, + ) + ), + graduate_denominator=float(np.dot(head_own + spouse_own + child_own, weight)), + new_borrower_weight=float( + weight[head_holds & head_new].sum() + + weight[spouse_holds & spouse_new].sum() + + weight[child_holds & child_new].sum() + ), + ) + + +def _racecl4_from_cps(person: pd.DataFrame) -> np.ndarray: + """SCF ``racecl4`` grouping of CPS race and Hispanic origin.""" + + race = pd.to_numeric(person["PRDTRACE"], errors="coerce").fillna(0).to_numpy() + hispanic = pd.to_numeric(person["PRDTHSP"], errors="coerce").fillna(0).to_numpy() + group = np.full(len(person), 4, dtype=np.int64) + group[race == 1] = 1 + group[race == 2] = 2 + group[hispanic > 0] = 3 + return group + + +def us_scf_education_debt_fit_table( + frame: Frame, + definition: RecipientDefinition = SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION, +) -> dict[str, object]: + """The microcosm#1073 fit table of an imputed US frame, at its weights. + + Households are grouped by the reference person's age and + ``racecl4``-equivalent race, and by weighted quintiles of the household + income the imputation conditions on (income quintiles of the frame's own + households, like the SCF's), all under ``definition``. + """ + + person = frame.table("person") + recipients = _recipients(frame, definition) + household_weight = np.asarray( + frame.weights_for("household").values, dtype=np.float64 + ) + own = ( + pd.to_numeric(person["federal_student_loan_balance"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + parent_plus = ( + pd.to_numeric(person["federal_parent_plus_loan_balance"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + graduate = ( + pd.to_numeric(person["federal_student_loan_graduate_share"], errors="coerce") + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + new_borrower = ( + person["is_income_based_repayment_new_borrower"].astype(bool).to_numpy() + ) + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + order = frame.table("household")["household_id"].to_numpy() + weight_by_household = pd.Series(household_weight, index=order) + person_weight = weight_by_household.reindex(household_ids).to_numpy( + dtype=np.float64 + ) + balance = own + parent_plus + household_debt = ( + pd.Series(balance, index=household_ids) + .groupby(level=0, sort=False) + .sum() + .reindex(order) + .fillna(0.0) + .to_numpy(dtype=np.float64) + ) + holds = balance > 0 + age = pd.to_numeric(person["age"], errors="coerce").fillna(0.0).to_numpy() + head = recipients.head_position + return education_debt_fit_table( + household_debt=household_debt, + household_weight=household_weight, + head_age=age[head], + race_group=_racecl4_from_cps(person)[head], + income_quintile=_income_quintiles( + recipients.features["household_income"].to_numpy(dtype=np.float64), + household_weight, + ), + borrower_balance=balance[holds], + borrower_age=age[holds], + borrower_weight=person_weight[holds], + parent_plus_total=float(np.dot(parent_plus, person_weight)), + graduate_numerator=float(np.dot(own * graduate, person_weight)), + graduate_denominator=float(np.dot(own, person_weight)), + new_borrower_weight=float(person_weight[holds & new_borrower].sum()), + ) + + +def us_scf_education_debt_summary(frame: Frame) -> dict[str, object]: + """Weighted signal and consistency diagnostics for the four columns.""" + + person = frame.table("person") + present = [c for c in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS if c in person.columns] + summary: dict[str, object] = { + "present_columns": present, + "household_income_columns_present": [ + c for c in _RECIPIENT_HOUSEHOLD_INCOME_COLUMNS if c in person.columns + ], + "borrower_household_share_band": list(_BORROWER_HOUSEHOLD_SHARE_BAND), + } + if len(present) != len(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS): + return summary + balances = { + column: pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64 + ) + for column in US_SCF_EDUCATION_DEBT_BALANCE_COLUMNS + } + graduate = pd.to_numeric( + person["federal_student_loan_graduate_share"], errors="coerce" + ).to_numpy(dtype=np.float64) + new_borrower = person["is_income_based_repayment_new_borrower"] + holds = ( + balances["federal_student_loan_balance"] + + balances["federal_parent_plus_loan_balance"] + ) > 0 + household_ids = person[_HOUSEHOLD_ID_COLUMN].to_numpy() + order = frame.table("household")["household_id"].to_numpy() + household_weight = np.asarray( + frame.weights_for("household").values, dtype=np.float64 + ) + household_holds = ( + pd.Series(holds, index=household_ids) + .groupby(level=0, sort=False) + .any() + .reindex(order) + .fillna(False) + .to_numpy(dtype=bool) + ) + total_weight = float(household_weight.sum()) + summary.update( + { + "nonfinite_counts": { + column: int((~np.isfinite(values)).sum()) + for column, values in balances.items() + }, + "negative_counts": { + column: int((values < 0).sum()) for column, values in balances.items() + }, + "unique_counts": { + column: int(pd.Series(values).nunique()) + for column, values in balances.items() + }, + "graduate_share_out_of_range": int( + ((graduate < 0) | (graduate > 1) | ~np.isfinite(graduate)).sum() + ), + "graduate_share_without_own_balance": int( + ((graduate > 0) & (balances["federal_student_loan_balance"] <= 0)).sum() + ), + "new_borrower_dtype_is_bool": bool( + pd.api.types.is_bool_dtype(new_borrower) + ), + "new_borrower_without_balance": int( + (new_borrower.astype(bool).to_numpy() & ~holds).sum() + ), + "borrower_household_share": ( + float(household_weight[household_holds].sum()) / total_weight + if total_weight > 0 + else 0.0 + ), + } + ) + return summary + + +def us_scf_education_debt_signal_gate(frame: Frame) -> GateResult: + """Require finite, nonnegative, nonconstant and coherent education debt.""" + + summary = us_scf_education_debt_summary(frame) + missing = sorted( + set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) - set(summary["present_columns"]) + ) + if missing: + return GateResult( + name="scf_education_debt_signal", + passed=False, + failures=(f"person columns missing: {missing}.",), + details=summary, + ) + failures: list[str] = [] + for column, count in summary["nonfinite_counts"].items(): + if count: + failures.append(f"{column}: {count} nonfinite value(s).") + for column, count in summary["negative_counts"].items(): + if count: + failures.append(f"{column}: {count} negative value(s).") + for column, count in summary["unique_counts"].items(): + if count < 2: + failures.append(f"{column}: constant column carries no signal.") + if summary["graduate_share_out_of_range"]: + failures.append( + "federal_student_loan_graduate_share: " + f"{summary['graduate_share_out_of_range']} value(s) outside [0, 1]." + ) + if summary["graduate_share_without_own_balance"]: + failures.append( + "federal_student_loan_graduate_share is positive for " + f"{summary['graduate_share_without_own_balance']} person(s) without " + "a federal_student_loan_balance." + ) + if not summary["new_borrower_dtype_is_bool"]: + failures.append("is_income_based_repayment_new_borrower is not boolean.") + if summary["new_borrower_without_balance"]: + failures.append( + "is_income_based_repayment_new_borrower is true for " + f"{summary['new_borrower_without_balance']} person(s) without a " + "federal balance." + ) + share = float(summary["borrower_household_share"]) + low, high = _BORROWER_HOUSEHOLD_SHARE_BAND + if not low <= share <= high: + failures.append( + f"federal education-debt household share {share:.3f} outside " + f"plausibility band [{low}, {high}]." + ) + return GateResult( + name="scf_education_debt_signal", + passed=not failures, + failures=tuple(failures), + details=summary, + ) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 7be8062b02..6a13410e86 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_184 - assert fields["authored_normative_field_count"] == 32_404 - assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_184 + assert fields["configuration_field_count"] == 42_368 + assert fields["authored_normative_field_count"] == 32_584 + assert fields["resolved_binding_field_count"] == 9_784 + assert fields["consumed_field_count"] == 42_368 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_182, + "compiler_semantic": 27_729, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_670, + "no_generation0_effect": 3_698, } inventory = coverage_report["inventory_coverage"] @@ -107,7 +107,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["missing_items"] == [] assert inventory["counts"]["producer_inputs"] == 2_744 assert inventory["counts"]["ownership_rows"] == 18 - assert inventory["counts"]["tail_control_fields"] == 934 + assert inventory["counts"]["tail_control_fields"] == 938 assert inventory["counts"]["seed_owner_bindings"] == 112 diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py index ccec9ee148..4fa9ac6022 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py @@ -31,7 +31,7 @@ ROOT = _TEST_PATHS.repository US_ROOT = ROOT / "packages/microcosm-build/src/microcosm/build/us" CANONICAL_LEGACY_PROJECTION_SHA256 = ( - "05cb099ee8bd0234ae94586b76dc20ab8a5bd0735c00d4e82e67f912e0b33823" + "a086632ea6fed35b4d8010f141f4d317aa5ab43055cafad44ca2cae35e2cebca" ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py index b025205ae5..e9545c0a4a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py @@ -143,7 +143,7 @@ def test_ir_retains_all_typed_resolution_products(compiled_us: CompiledSpecIR) - assert len(inventory["columns"]) == 176 assert len(inventory["artifacts"]) == 84 assert len(inventory["scopes"]) == 7 - assert len(inventory["references"]) == 334 + assert len(inventory["references"]) == 336 assert "engine_abi_lock" in compiled_us.generated_authorities assert "records" in compiled_us.vintage_authorities assert set(compiled_us.surfaces) == { diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index c5ec18bb51..3efc9dbaaf 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_184 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_368 assert field_ledger.source_counts == { - "authored": 32_404, - "resolved_bindings": 9_780, + "authored": 32_584, + "resolved_bindings": 9_784, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_182, + "compiler_semantic": 27_729, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_670, + "no_generation0_effect": 3_698, } - assert len({field.pointer for field in field_ledger.fields}) == 42_184 + assert len({field.pointer for field in field_ledger.fields}) == 42_368 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index a780c4814b..f6215aae11 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf", + "sha256": "5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" @@ -168,7 +168,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: sources = build_sources() vintages = build_vintages() records = vintages["records"] - assert len(records) == 16 + assert len(records) == 17 by_id = {record["id"]: record for record in records} assert set(by_id) == { "acs_2022", @@ -184,6 +184,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: "puma_2020", "release_us_2024", "scf_2022", + "scf_2025", "sipp_2023", "target_2024", "tax_2015", @@ -204,7 +205,7 @@ def test_all_us_vintages_have_reciprocal_reviewed_compatibility() -> None: } ) resolved_wire = thaw_json(resolved.vintage_authorities) - assert len(resolved_wire["records"]) == 16 + assert len(resolved_wire["records"]) == 17 assert ( resolved_wire["records"]["policyengine_us_surface"]["value"] == (ENGINE_LOCK["engine"]["version"]) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 4fa27b499d..bb7885d11f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -2082,6 +2082,108 @@ def test_builder_exact_k_parser_enforces_charter_and_explicit_seed(capsys) -> No assert parsed.no_staging is True +def test_scf_education_debt_stage_is_opt_in_and_follows_auto_loans() -> None: + """microcosm#1073: the 2025 SCF stage runs only when asked, after auto loans. + + The pinned engine does not define the four federal student loan inputs + yet, so a default release must not fetch the 2025 SCF or carry them. + """ + + builder = _load_builder_module() + base_argv = ["--ledger-facts", "facts.jsonl", "--out", "release"] + parsed = builder._parse_args(base_argv) + assert parsed.scf_education_debt is False + assert parsed.scf_2025_summary_extract is None + assert parsed.scf_2025_full_extract is None + + parsed = builder._parse_args( + [ + *base_argv, + "--scf-education-debt", + "--scf-2025-summary-extract", + "rscfp2025.dta", + "--scf-2025-full-extract", + "p25i6.dta", + ] + ) + assert parsed.scf_education_debt is True + assert parsed.scf_2025_summary_extract == Path("rscfp2025.dta") + assert parsed.scf_2025_full_extract == Path("p25i6.dta") + + source = inspect.getsource(builder._main) + gate = source.index("if args.scf_education_debt:") + assert source.index("with_us_scf_auto_loan_inputs(") < gate + assert gate < source.index("_with_scf_education_debt_stage(") + assert source.index("join_acs_release_predictors(") < gate + + +def test_scf_education_debt_stage_runs_its_gate_and_writes_its_fit( + tmp_path, monkeypatch +) -> None: + builder = _load_builder_module() + calls: list[str] = [] + frame = object() + + monkeypatch.setattr( + builder, + "load_scf_2025_education_debt_donor", + lambda summary, full: calls.append(f"load:{summary}:{full}") or "donor", + ) + + def fake_with_inputs(base_frame, *, seed, time_period, scf_education_debt_donor): + assert base_frame is frame + assert scf_education_debt_donor == "donor" + calls.append(f"impute:{seed}:{time_period}") + return base_frame + + monkeypatch.setattr(builder, "with_us_scf_education_debt_inputs", fake_with_inputs) + monkeypatch.setattr( + builder, + "us_scf_education_debt_signal_gate", + lambda base_frame: builder.GateResult( + name="scf_education_debt_signal", passed=True + ), + ) + monkeypatch.setattr( + builder, + "us_scf_education_debt_fit_table", + lambda base_frame: {"households": {"share_with_federal_education_debt": 0.2}}, + ) + args = SimpleNamespace( + seed=7, + scf_2025_summary_extract=Path("rscfp2025.dta"), + scf_2025_full_extract=Path("p25i6.dta"), + ) + fit_path = tmp_path / "artifacts" / "us_scf_education_debt_fit.json" + assert ( + builder._with_scf_education_debt_stage( + frame, args=args, telemetry=None, fit_path=fit_path + ) + is frame + ) + assert calls == [ + f"load:{Path('rscfp2025.dta')}:{Path('p25i6.dta')}", + f"impute:7:{builder.PERIOD}", + ] + payload = json.loads(fit_path.read_text(encoding="utf-8")) + assert payload["weights"] == "pre_calibration_household_weights" + assert payload["fit"]["households"]["share_with_federal_education_debt"] == 0.2 + + monkeypatch.setattr( + builder, + "us_scf_education_debt_signal_gate", + lambda base_frame: builder.GateResult( + name="scf_education_debt_signal", + passed=False, + failures=("constant column",), + ), + ) + with pytest.raises(RuntimeError, match="SCF education-debt signal failed"): + builder._with_scf_education_debt_stage( + frame, args=args, telemetry=None, fit_path=None + ) + + def test_builder_gate_failed_base_pool_override_is_explicit(capsys) -> None: builder = _load_builder_module() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py new file mode 100644 index 0000000000..d276c72f9a --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_education_debt.py @@ -0,0 +1,1123 @@ +"""SCF 2025 federal student loan stage tests (microcosm#1073). + +Federal education debt reads the 2025 SCF: per-loan federal flags, whose +education each loan paid for, the year borrowed and the program financed. The +stage writes four person inputs that policyengine-us#9721 and #9724 add. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import zipfile + +import numpy as np +import pandas as pd +import pytest + +import microcosm.build.us_runtime.scf_education_debt as education_debt +from microcosm.build.us_runtime.scf_education_debt import ( + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + SCF_EDUCATION_DEBT_COMPOSITION_TARGETS, + SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS, + SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, + SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN, + SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + SCF_EDUCATION_LOAN_YEAR_COLUMNS, + SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, + ScfPublicFile, + derive_scf_education_debt_targets, + draw_scf_education_debt_households, + education_debt_fit_table, + fetch_scf_public_file, + impute_us_scf_education_debt, + load_scf_2025_education_debt_donor, + scf_education_debt_composition_predictors, + scf_education_debt_donor_fit_table, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, + us_scf_education_debt_stage_spec, + us_scf_education_debt_summary, + with_us_scf_education_debt_inputs, +) +from microcosm.build.us_runtime.scf_wealth import SCF_WEALTH_PREDICTORS +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +TIME_PERIOD = 2024 +_DONOR_WEIGHT_COLUMN = "scf_weight" +_LOAN_ITEMS = { + "balance": SCF_EDUCATION_LOAN_BALANCE_COLUMNS, + "federal": SCF_EDUCATION_LOAN_FEDERAL_COLUMNS, + "whose": SCF_EDUCATION_LOAN_WHOSE_COLUMNS, + "year": SCF_EDUCATION_LOAN_YEAR_COLUMNS, + "program": SCF_EDUCATION_LOAN_PROGRAM_COLUMNS, + "borrowed": SCF_EDUCATION_LOAN_AMOUNT_BORROWED_COLUMNS, +} + + +def _family( + *loans: dict[str, float], + remaining: float = 0.0, + reversed_codes: bool = False, + listed: tuple[tuple[int, int], ...] = (), +) -> dict[str, float]: + """One full-file row: up to six loans given as item dictionaries. + + ``listed`` gives household-listing persons 3-12 as (relationship, public + age code) pairs; ``reversed_codes`` sets X8000 = 1. + """ + + row = {column: 0.0 for columns in _LOAN_ITEMS.values() for column in columns} + row[SCF_EDUCATION_LOAN_REMAINING_BALANCE_COLUMN] = remaining + row["x8000"] = 1.0 if reversed_codes else 5.0 + for column in ( + *SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS, + *SCF_HOUSEHOLD_LISTING_AGE_COLUMNS, + ): + row[column] = 0.0 + for index, (relationship, age_code) in enumerate(listed): + row[SCF_HOUSEHOLD_LISTING_RELATIONSHIP_COLUMNS[index]] = float(relationship) + row[SCF_HOUSEHOLD_LISTING_AGE_COLUMNS[index]] = float(age_code) + for index, loan in enumerate(loans): + for item, value in loan.items(): + row[_LOAN_ITEMS[item][index]] = value + return row + + +def _loan( + balance: float, + *, + federal: int = 1, + whose: int = 1, + year: int = 2018, + program: int = 12, + borrowed: float | None = None, +) -> dict[str, float]: + return { + "balance": balance, + "federal": federal, + "whose": whose, + "year": year, + "program": program, + "borrowed": balance if borrowed is None else borrowed, + } + + +def _derive(*families: dict[str, float]) -> pd.DataFrame: + return derive_scf_education_debt_targets(pd.DataFrame.from_records(families)) + + +# --------------------------------------------------------------------------- # +# Donor targets # +# --------------------------------------------------------------------------- # +def test_only_federal_loans_count_and_holders_follow_whose_education() -> None: + row = _derive( + _family( + _loan(10_000, whose=1, year=2016, program=12), + _loan(20_000, whose=2, year=2010, program=13, borrowed=25_000), + _loan(5_000, whose=3, year=2019, program=12), + _loan(7_000, federal=5, whose=1), + _loan(3_000, federal=0, whose=1), + ) + ).iloc[0] + + assert row["head_federal_student_loan_balance"] == 10_000 + assert row["spouse_federal_student_loan_balance"] == 20_000 + assert row["federal_parent_plus_loan_balance"] == 5_000 + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 35_000 + assert row["head_own_share"] == pytest.approx(10_000 / 35_000) + assert row["spouse_own_share"] == pytest.approx(20_000 / 35_000) + # The head holds the 2016 own loan and the 2019 Parent PLUS loan. + assert row["head_is_ibr_new_borrower"] == 1.0 + assert row["spouse_is_ibr_new_borrower"] == 0.0 + assert row["head_graduate_share"] == 0.0 + assert row["spouse_graduate_share"] == 1.0 + assert row["education_debt"] == 45_000 + assert row["federal_status_unrecorded_balance"] == 3_000 + + +def test_remaining_loans_take_the_family_federal_share_and_holder_split() -> None: + row = _derive( + _family( + _loan(6_000, whose=1), + _loan(4_000, federal=5, whose=1), + _loan(2_000, whose=3), + remaining=6_000, + ) + ).iloc[0] + + # Loans 1-6 total 12,000, so each is scaled by 1 + 6,000 / 12,000. + assert row["head_federal_student_loan_balance"] == pytest.approx(9_000) + assert row["federal_parent_plus_loan_balance"] == pytest.approx(3_000) + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == pytest.approx(12_000) + assert row["education_debt"] == pytest.approx(18_000) + assert row["remaining_loans_without_detail"] == 0.0 + + +def test_remaining_loans_without_detail_are_dropped_and_reported() -> None: + row = _derive(_family(remaining=4_000)).iloc[0] + + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 0.0 + assert row["remaining_loans_without_detail"] == 4_000 + assert row["head_own_share"] == 0.0 + assert row["head_is_ibr_new_borrower"] == 0.0 + + +@pytest.mark.parametrize("code", [6, 7, -7, 0, 4]) +def test_multiple_unrelated_other_and_unknown_codes_go_to_the_head(code) -> None: + row = _derive(_family(_loan(8_000, whose=code))).iloc[0] + + assert row["head_federal_student_loan_balance"] == 8_000 + assert row["spouse_federal_student_loan_balance"] == 0.0 + assert row["federal_parent_plus_loan_balance"] == 0.0 + assert row["head_own_share"] == 1.0 + + +def test_graduate_share_weights_by_amount_borrowed_with_balance_fallback() -> None: + weighted = _derive( + _family( + _loan(5_000, program=14, borrowed=30_000), + _loan(9_000, program=11, borrowed=10_000), + ) + ).iloc[0] + fallback = _derive( + _family( + _loan(5_000, program=15, borrowed=0.0), + _loan(15_000, program=16, borrowed=0.0), + ) + ).iloc[0] + parent_plus_only = _derive(_family(_loan(5_000, whose=3, program=13))).iloc[0] + + assert weighted["head_graduate_share"] == pytest.approx(0.75) + assert fallback["head_graduate_share"] == pytest.approx(0.25) + # A Parent PLUS loan is not the parent's own loan. + assert parent_plus_only["head_graduate_share"] == 0.0 + + +def test_new_borrower_status_counts_2014_as_new_and_needs_every_loan() -> None: + rows = _derive( + _family(_loan(1_000, year=2014)), + _family(_loan(1_000, year=2013)), + _family(_loan(1_000, year=2020), _loan(1_000, year=2005, whose=3)), + _family(_loan(1_000, year=2020), _loan(1_000, year=2005, federal=5)), + ) + + assert rows["head_is_ibr_new_borrower"].tolist() == [1.0, 0.0, 0.0, 1.0] + + +def test_derivation_refuses_a_missing_loan_item() -> None: + frame = pd.DataFrame.from_records([_family(_loan(1_000))]).drop(columns="x9338") + with pytest.raises(ValueError, match="x9338"): + derive_scf_education_debt_targets(frame) + listing = pd.DataFrame.from_records([_family(_loan(1_000))]).drop(columns="x8000") + with pytest.raises(ValueError, match="x8000"): + derive_scf_education_debt_targets(listing) + + +def test_reversed_families_swap_the_codes_of_loans_4_and_6_back() -> None: + loans = [_loan(1_000.0 * (index + 1), whose=1) for index in range(6)] + reversed_row, unreversed_row = _derive( + _family(*loans, reversed_codes=True), _family(*loans) + ).to_dict("records") + + # Loans 4 and 6 (4,000 and 6,000) still name the original respondent, now + # the spouse or partner, in a reversed family; loans 1, 2, 3 and 5 do not. + assert reversed_row["head_federal_student_loan_balance"] == 11_000 + assert reversed_row["spouse_federal_student_loan_balance"] == 10_000 + assert unreversed_row["head_federal_student_loan_balance"] == 21_000 + assert unreversed_row["spouse_federal_student_loan_balance"] == 0.0 + spouse_coded = _derive( + _family( + _loan(1_000, whose=2), + _loan(1_000, whose=2), + _loan(1_000, whose=2), + _loan(4_000, whose=2), + _loan(1_000, whose=3), + _loan(6_000, whose=2), + reversed_codes=True, + ) + ).iloc[0] + assert spouse_coded["head_federal_student_loan_balance"] == 10_000 + assert spouse_coded["spouse_federal_student_loan_balance"] == 3_000 + # Code 3 names neither partner, so it is never swapped. + assert spouse_coded["federal_parent_plus_loan_balance"] == 1_000 + + +_CHILD = 4 +_GRANDCHILD = 5 + + +@pytest.mark.parametrize( + ("listed", "loan", "holder"), + [ + # A child aged 18-27 (code 25) who could have borrowed it in 2022. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2022), "child"), + # No listed child: the student lives elsewhere. + ((), _loan(9_000, whose=3, year=2022), "parent_plus"), + # Only a 13-17-year-old (code 17) and a grandchild. + (((_CHILD, 17), (_GRANDCHILD, 25)), _loan(9_000, whose=3, year=2022), + "parent_plus"), + # At most 27 now, so at most 2 when a 2000 loan was borrowed. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2000), "parent_plus"), + # Above the $31,000 dependent limit while possibly dependent. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2022, borrowed=40_000), + "parent_plus"), + # At the limit itself, which the child's own loans can reach. + (((_CHILD, 25),), _loan(9_000, whose=3, year=2022, borrowed=31_000), + "child"), + # A graduate loan is never Parent PLUS, whatever its size. + (((_CHILD, 25),), + _loan(9_000, whose=3, year=2022, program=13, borrowed=80_000), "child"), + # A child aged 38-42 (code 40) was at least 28 in 2015: independent. + (((_CHILD, 40),), _loan(9_000, whose=3, year=2015, borrowed=50_000), + "child"), + # Aged 28-32 (code 30), so possibly 23 in 2018: possibly dependent. + (((_CHILD, 30),), _loan(9_000, whose=3, year=2018, borrowed=50_000), + "parent_plus"), + ], +) # fmt: skip +def test_loans_for_a_child_go_to_a_co_resident_adult_child_who_fits( + listed, loan, holder +) -> None: + row = _derive(_family(loan, listed=listed)).iloc[0] + + child = row["child_federal_student_loan_balance"] + parent_plus = row["federal_parent_plus_loan_balance"] + expected = (9_000, 0.0) if holder == "child" else (0.0, 9_000) + assert (child, parent_plus) == expected + assert row["child_own_share"] + row["parent_plus_share"] == 1.0 + assert row["head_federal_student_loan_balance"] == 0.0 + + +def test_child_targets_carry_the_childs_flags_shares_and_ages() -> None: + row = _derive( + _family( + _loan(10_000, whose=1, year=2010), + _loan(6_000, whose=3, year=2020, program=12), + _loan(4_000, whose=3, year=2023, program=13, borrowed=12_000), + _loan(2_000, whose=3, year=2001), + listed=((_CHILD, 30), (_CHILD, 25), (_CHILD, 12)), + ) + ).iloc[0] + + assert row["child_federal_student_loan_balance"] == 10_000 + # The 2001 loan fits no listed adult child (at most 32 now, 8 then). + assert row["federal_parent_plus_loan_balance"] == 2_000 + assert row[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 22_000 + assert row["child_own_share"] == pytest.approx(10_000 / 22_000) + assert row["child_graduate_share"] == pytest.approx(12_000 / 18_000) + assert row["child_is_ibr_new_borrower"] == 1.0 + # The head holds a 2010 loan and the 2001 Parent PLUS loan. + assert row["head_is_ibr_new_borrower"] == 0.0 + assert row[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] == 2.0 + assert row["youngest_adult_child_age_code"] == 25.0 + assert row["child_borrower_age_code"] == 25.0 + shares = [f"{holder}_own_share" for holder in ("head", "spouse", "child")] + assert row[[*shares, "parent_plus_share"]].sum() == pytest.approx(1.0) + + +# --------------------------------------------------------------------------- # +# Loader # +# --------------------------------------------------------------------------- # +def _summary_and_full(n_families: int = 6) -> tuple[pd.DataFrame, pd.DataFrame]: + summary_rows: list[dict[str, float]] = [] + full_rows: list[dict[str, float]] = [] + for family in range(1, n_families + 1): + loans = [_loan(1_000.0 * family, whose=1 + family % 3)] if family % 2 else [] + for implicate in range(1, 6): + y1 = family * 10 + implicate + full = {"y1": float(y1), "yy1": float(family), "x19": 40.0 + family} + full.update(_family(*loans)) + full_rows.append(full) + debt = sum(loan["balance"] for loan in loans) + summary_rows.append( + { + "y1": float(y1), + "yy1": float(family), + "wgt": 1_000.0 + family, + "age": 25.0 + family, + "hhsex": 1.0 + family % 2, + "racecl5": float(family % 5 + 1), + "racecl4": float(family % 4 + 1), + "married": float(family % 2), + "kids": float(family % 3), + "wageinc": 30_000.0 + family, + "intdivinc": 100.0 * family, + "ssretinc": 0.0, + "income": 40_000.0 + 1_000.0 * family, + "kginc": 500.0 * family, + "edn_inst": debt, + "educ": 12.0, + } + ) + return pd.DataFrame(summary_rows), pd.DataFrame(full_rows).iloc[::-1] + + +def _write(tmp_path, summary: pd.DataFrame, full: pd.DataFrame): + summary_path = tmp_path / "rscfp2025.dta" + full_path = tmp_path / "p25i6.dta" + summary.to_stata(summary_path, write_index=False) + full.to_stata(full_path, write_index=False) + return summary_path, full_path + + +def test_loader_joins_both_files_and_derives_targets_and_predictors(tmp_path) -> None: + summary, full = _summary_and_full() + donor = load_scf_2025_education_debt_donor( + *_write(tmp_path, summary, full), expected_rows=None + ) + + assert len(donor) == len(summary) + assert set(SCF_EDUCATION_DEBT_PREDICTORS) <= set(donor.columns) + assert set(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS) <= set(donor.columns) + first = donor.iloc[0] + assert first["household_income"] == pytest.approx(41_000.0 - 500.0) + assert first["spouse_age"] == 41.0 + assert sorted(donor["implicate"].unique()) == [1, 2, 3, 4, 5] + family_three = donor.loc[donor["yy1"] == 3].iloc[0] + assert family_three[SCF_EDUCATION_DEBT_TOTAL_TARGET] == 3_000.0 + assert family_three["head_federal_student_loan_balance"] == 3_000.0 + + +def test_loader_refuses_drifted_education_debt_and_row_counts(tmp_path) -> None: + summary, full = _summary_and_full() + with pytest.raises(ValueError, match="expected 30"): + load_scf_2025_education_debt_donor( + *_write(tmp_path, summary.iloc[:-5], full.iloc[:-5]), expected_rows=30 + ) + drifted = summary.copy() + drifted.loc[0, "edn_inst"] = 99_999.0 + with pytest.raises(ValueError, match="edn_inst"): + load_scf_2025_education_debt_donor( + *_write(tmp_path, drifted, full), expected_rows=None + ) + + +def test_loader_refuses_an_unmatched_implicate(tmp_path) -> None: + summary, full = _summary_and_full() + full = full.copy() + full.iloc[0, full.columns.get_loc("y1")] = 999.0 + with pytest.raises(ValueError, match="unmatched"): + load_scf_2025_education_debt_donor( + *_write(tmp_path, summary, full), expected_rows=None + ) + + +# --------------------------------------------------------------------------- # +# Declaration and source files # +# --------------------------------------------------------------------------- # +def test_stage_declares_the_person_outputs_and_pinned_2025_files() -> None: + spec = us_scf_education_debt_stage_spec() + + assert set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) <= set(spec.outputs) + locators = {artifact.get("locator"): artifact for artifact in spec.artifacts} + for public_file in (SCF_2025_SUMMARY_EXTRACT, SCF_2025_FULL_EXTRACT): + artifact = locators[public_file.url] + assert artifact["vintage"] == "2025" + assert artifact["sha256"] == public_file.zip_sha256 + assert artifact["member_sha256"] == public_file.member_sha256 + assert artifact["expected_rows"] == public_file.rows + # Net worth and the other wealth outputs stay on the 2022 files. + assert spec.artifacts[0]["vintage"] == "2022" + assert spec.artifacts[1]["vintage"] == "2022" + assert spec.artifacts[2]["member"] == "pu2023.csv" + # Both QRFs condition on the production predictors; the borrower fit adds + # the adult children and the drawn total. + fits = { + operation.parameters["donor"]: operation.parameters + for operation in spec.operations + if operation.kind == "fit_weighted_qrf" + and str(operation.parameters.get("donor", "")).startswith("scf_2025") + } + assert tuple(fits["scf_2025_education_debt"]["predictors"]) == ( + SCF_EDUCATION_DEBT_PREDICTORS + ) + assert tuple(fits["scf_2025_education_debt_borrowers"]["predictors"]) == ( + *SCF_EDUCATION_DEBT_PREDICTORS, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + SCF_EDUCATION_DEBT_TOTAL_TARGET, + ) + assert scf_education_debt_composition_predictors() == tuple( + fits["scf_2025_education_debt_borrowers"]["predictors"] + ) + assert tuple(fits["scf_2025_education_debt_borrowers"]["targets"]) == ( + SCF_EDUCATION_DEBT_COMPOSITION_TARGETS + ) + # Parent PLUS is drawn first, for itself, not left as a residual. + assert SCF_EDUCATION_DEBT_COMPOSITION_TARGETS[0] == "parent_plus_share" + + +def test_production_predictors_drop_only_the_reviewed_exclusions() -> None: + candidates = education_debt.SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS + excluded = education_debt.SCF_EDUCATION_DEBT_EXCLUDED_PREDICTORS + assert set(excluded) == { + "interest_dividend_income", + "social_security_pension_income", + } + assert all(reason.strip() for reason in excluded.values()) + assert SCF_EDUCATION_DEBT_PREDICTORS == tuple( + predictor for predictor in candidates if predictor not in excluded + ) + assert education_debt.SCF_EDUCATION_DEBT_RECIPIENT_DEFINITION == ( + education_debt.RecipientDefinition( + reference_person="householder_couple", + partnered="spouse_or_partner", + children="of_reference_or_partner", + income_unit="reference_couple_and_children", + ) + ) + + +def _zip_with(member: str, payload: bytes) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(member, payload) + return buffer.getvalue() + + +class _Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_fetch_verifies_both_pins_and_reuses_a_verified_cache( + tmp_path, monkeypatch +) -> None: + member = b"stata bytes" + archive = _zip_with("p25i6.dta", member) + pinned = ScfPublicFile( + url="https://example.invalid/scf2025s.zip", + member="p25i6.dta", + zip_sha256=hashlib.sha256(archive).hexdigest(), + member_sha256=hashlib.sha256(member).hexdigest(), + rows=1, + ) + fetched: list[str] = [] + + def fake_urlopen(request, timeout): + fetched.append(request.full_url) + return _Response(archive) + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + path = fetch_scf_public_file(pinned, tmp_path) + assert path.read_bytes() == member + assert fetch_scf_public_file(pinned, tmp_path) == path + assert fetched == [pinned.url] + + wrong_zip = ScfPublicFile( + url=pinned.url, + member=pinned.member, + zip_sha256="0" * 64, + member_sha256=pinned.member_sha256, + rows=1, + ) + with pytest.raises(ValueError, match="sha-256"): + fetch_scf_public_file(wrong_zip, tmp_path / "other") + + +# --------------------------------------------------------------------------- # +# Imputation # +# --------------------------------------------------------------------------- # +def _ready_donor(n: int = 600) -> pd.DataFrame: + rng = np.random.default_rng(1073) + donor = pd.DataFrame({name: rng.normal(size=n) for name in SCF_WEALTH_PREDICTORS}) + donor["age"] = rng.integers(20, 85, n).astype(np.float64) + donor["is_female"] = rng.integers(0, 2, n).astype(np.float64) + donor["cps_race"] = rng.integers(1, 8, n).astype(np.float64) + donor["is_married"] = rng.integers(0, 2, n).astype(np.float64) + donor["own_children_in_household"] = rng.integers(0, 3, n).astype(np.float64) + donor["employment_income"] = rng.gamma(2.0, 25_000.0, n) + donor["interest_dividend_income"] = rng.gamma(1.0, 800.0, n) + donor["social_security_pension_income"] = rng.gamma(1.0, 5_000.0, n) + donor["household_income"] = donor["employment_income"] + rng.gamma(1.0, 5_000.0, n) + donor[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = np.where( + donor["age"] >= 40, rng.integers(0, 3, n), 0 + ).astype(np.float64) + borrower = (rng.random(n) < 0.2) & (donor["age"] < 65) + total = np.where(borrower, rng.gamma(1.5, 25_000.0, n).round(), 0.0) + # Holders: 0 head, 1 spouse, 2 Parent PLUS, 3 an adult child. + holder = rng.choice(4, n, p=[0.55, 0.25, 0.1, 0.1]) + holder = np.where( + (holder == 3) & (donor[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] == 0), + 0, + holder, + ) + for index, column in enumerate( + ("head_own_share", "spouse_own_share", "parent_plus_share", "child_own_share") + ): + donor[column] = np.where(borrower, (holder == index).astype(float), 0.0) + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = total + for prefix, choices in ( + ("head", [0.0, 0.5, 1.0]), + ("spouse", [0.0, 1.0]), + ("child", [0.0, 1.0]), + ): + donor[f"{prefix}_graduate_share"] = np.where( + donor[f"{prefix}_own_share"] > 0, rng.choice(choices, n), 0.0 + ) + head_holds = (donor["head_own_share"] + donor["parent_plus_share"]) > 0 + for prefix, holds in ( + ("head", head_holds), + ("spouse", donor["spouse_own_share"] > 0), + ("child", donor["child_own_share"] > 0), + ): + donor[f"{prefix}_is_ibr_new_borrower"] = np.where( + holds, (rng.random(n) < 0.5).astype(float), 0.0 + ) + donor[_DONOR_WEIGHT_COLUMN] = rng.uniform(500.0, 2_000.0, n) + return donor + + +def _features(donor: pd.DataFrame) -> pd.DataFrame: + return donor.loc[ + :, [*SCF_EDUCATION_DEBT_PREDICTORS, SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] + ] + + +def _person_rows(n_households: int = 150, *, adult_children: bool = False): + """Couples (A_SPOUSE pointers) and single adults with a child. + + With ``adult_children``, every couple's child is 19 to 24 and every fourth + couple also has a 30-year-old child (line 4). + """ + + rng = np.random.default_rng(97) + records: list[dict[str, object]] = [] + person_id = 1 + for household_id in range(1, n_households + 1): + couple = household_id % 3 != 0 + members = ( + [(1, 2, 1, 0), (2, 1, 1, 1), (3, 0, 7, 0)] + if couple + else [(1, 0, 7, 0), (2, 0, 7, 1)] + ) + if adult_children and couple and household_id % 4 == 0: + members.append((4, 0, 7, 1)) + for line, spouse, marital, female in members: + child = line >= 3 or (not couple and line == 2) + if not child: + age = float(rng.integers(24, 60)) + elif adult_children and couple: + age = 30.0 if line == 4 else float(rng.integers(19, 25)) + else: + age = float(rng.integers(5, 17)) + records.append( + { + "person_id": person_id, + "person_household_id": household_id, + "PH_SEQ": household_id, + "A_LINENO": line, + "A_SPOUSE": spouse, + "age": age, + "is_female": bool(female), + "PRDTRACE": int(rng.integers(1, 5)), + "PRDTHSP": int(rng.integers(0, 2)), + "A_MARITL": marital, + "PEPAR1": 1 if child else -1, + "PEPAR2": -1, + "employment_income_before_lsr": 0.0 + if child + else float(rng.gamma(2.0, 20_000.0)), + "taxable_interest_income": float(rng.gamma(1.0, 300.0)), + } + ) + person_id += 1 + return pd.DataFrame.from_records(records) + + +def _us_frame(person: pd.DataFrame) -> Frame: + person = person.copy() + household_ids = np.sort(person["person_household_id"].unique()) + n_person = len(person) + person["person_tax_unit_id"] = person["person_household_id"] + 1_000 + person["person_spm_unit_id"] = person["person_household_id"] + 2_000 + person["person_family_id"] = person["person_household_id"] + 3_000 + person["person_marital_unit_id"] = np.arange(n_person) + 4_000 + tables = { + "person": person, + "household": pd.DataFrame({"household_id": household_ids}), + "tax_unit": pd.DataFrame({"tax_unit_id": household_ids + 1_000}), + "spm_unit": pd.DataFrame({"spm_unit_id": household_ids + 2_000}), + "family": pd.DataFrame({"family_id": household_ids + 3_000}), + "marital_unit": pd.DataFrame({"marital_unit_id": np.arange(n_person) + 4_000}), + } + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + values=np.full(len(household_ids), 1_000.0), + kind=WeightKind.DESIGN, + ) + }, + ) + + +def test_household_draws_are_seeded_coherent_and_nonnegative() -> None: + donor = _ready_donor() + features = _features(donor).iloc[:200] + first = draw_scf_education_debt_households(features, donor, seed=3, n_estimators=15) + second = draw_scf_education_debt_households( + features, donor, seed=3, n_estimators=15 + ) + pd.testing.assert_frame_equal(first, second) + + total = first[SCF_EDUCATION_DEBT_TOTAL_TARGET].to_numpy() + parts = ( + first["head_federal_student_loan_balance"] + + first["spouse_federal_student_loan_balance"] + + first["child_federal_student_loan_balance"] + + first["federal_parent_plus_loan_balance"] + ).to_numpy() + assert (total >= 0).all() and (total > 0).any() + np.testing.assert_array_equal(parts, total) + borrowers = total > 0 + shares = first[list(SCF_EDUCATION_DEBT_HOLDER_SHARE_TARGETS)].sum(axis=1) + np.testing.assert_allclose(shares[borrowers], 1.0) + assert not first.loc[~borrowers, list(SCF_EDUCATION_DEBT_COMPOSITION_TARGETS)].any( + axis=None + ) + # A household without an adult child holds no child balance. + no_adult_child = features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] == 0 + assert (first.loc[no_adult_child, "child_federal_student_loan_balance"] == 0).all() + + +def test_a_donor_family_without_parent_plus_rarely_draws_one() -> None: + """Parent PLUS is drawn for itself, not left over by the own shares. + + Own debt splits between the two partners in fractions, and only families + with a reference person 55 or older hold Parent PLUS. The draw for every + donor family's own predictors gives Parent PLUS to few of the others. + """ + + rng = np.random.default_rng(1074) + donor = _ready_donor(1_500) + borrower = donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0 + older = donor["age"] >= 55 + head_share = rng.uniform(0.2, 0.8, len(donor)) + donor["head_own_share"] = np.where(borrower & ~older, head_share, 0.0) + donor["spouse_own_share"] = np.where(borrower & ~older, 1.0 - head_share, 0.0) + donor["parent_plus_share"] = np.where(borrower & older, 1.0, 0.0) + donor["child_own_share"] = 0.0 + donor["child_graduate_share"] = 0.0 + donor["child_is_ibr_new_borrower"] = 0.0 + drawn = draw_scf_education_debt_households( + _features(donor), donor, seed=8, n_estimators=20 + ) + with_debt = drawn[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0 + draws_plus = drawn["federal_parent_plus_loan_balance"] > 0 + assert draws_plus[with_debt & ~older].mean() < 0.02 + assert draws_plus[with_debt & older].mean() > 0.9 + # A donor with no Parent PLUS at all never draws it. + donor["parent_plus_share"] = 0.0 + donor["head_own_share"] = np.where(borrower, head_share, 0.0) + donor["spouse_own_share"] = np.where(borrower, 1.0 - head_share, 0.0) + none = draw_scf_education_debt_households( + _features(donor), donor, seed=9, n_estimators=20 + ) + assert (none["federal_parent_plus_loan_balance"] == 0).all() + + +def test_imputation_reaches_only_reference_persons_spouses_and_adult_children() -> None: + frame = _us_frame(_person_rows()) + imputed = impute_us_scf_education_debt( + frame, _ready_donor(), seed=5, n_estimators=15 + ) + person = frame.table("person") + + assert list(imputed.columns) == list(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) + adult = person["age"] >= 18 + balance = ( + imputed["federal_student_loan_balance"] + + imputed["federal_parent_plus_loan_balance"] + ) + assert (balance >= 0).all() and (balance > 0).any() + assert (balance[~adult] == 0).all() + # Parent PLUS sits on the reference person (line 1 in these fixtures). + assert ( + imputed.loc[person["A_LINENO"] != 1, "federal_parent_plus_loan_balance"] == 0 + ).all() + own = imputed["federal_student_loan_balance"] + assert (imputed.loc[own <= 0, "federal_student_loan_graduate_share"] == 0).all() + assert imputed["federal_student_loan_graduate_share"].between(0, 1).all() + assert imputed["is_income_based_repayment_new_borrower"].dtype == bool + assert not ( + imputed["is_income_based_repayment_new_borrower"] & (balance <= 0) + ).any() + single_adults = person["A_MARITL"].eq(7) & adult + assert (person.loc[single_adults, "A_LINENO"] == 1).all() + + +def test_a_spouse_draw_without_a_spouse_moves_to_the_reference_person() -> None: + frame = _us_frame(_person_rows(30)) + donor = _ready_donor() + # Every donor family borrows, so every household draws a positive total and + # the test checks only where the spouse's draw lands. With the synthetic + # donor's random incidence, a 30-household frame otherwise draws no debt for + # any couple on about 2 seeds in 20. + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = np.maximum( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], 1_000.0 + ) + donor["head_own_share"] = 0.0 + donor["parent_plus_share"] = 0.0 + donor["child_own_share"] = 0.0 + donor["spouse_own_share"] = np.where( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, 1.0, 0.0 + ) + donor["head_graduate_share"] = 0.0 + donor["head_is_ibr_new_borrower"] = 0.0 + imputed = impute_us_scf_education_debt(frame, donor, seed=11, n_estimators=10) + person = frame.table("person") + + single = person["person_household_id"] % 3 == 0 + reference = person["A_LINENO"] == 1 + couple_spouse = ~single & (person["A_LINENO"] == 2) + own = imputed["federal_student_loan_balance"] + assert (own[single & ~reference] == 0).all() + assert (own[single & reference] > 0).any() + assert (own[~single & reference] == 0).all() + assert (own[couple_spouse] > 0).any() + assert (imputed["federal_parent_plus_loan_balance"] == 0).all() + + +def test_a_childs_draw_goes_to_the_youngest_adult_child() -> None: + frame = _us_frame(_person_rows(40, adult_children=True)) + donor = _ready_donor() + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] = np.maximum( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET], 1_000.0 + ) + for column in ("head_own_share", "spouse_own_share", "parent_plus_share"): + donor[column] = 0.0 + donor["child_own_share"] = 1.0 + donor["child_graduate_share"] = 1.0 + donor["child_is_ibr_new_borrower"] = 1.0 + donor["head_is_ibr_new_borrower"] = 0.0 + donor[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR] = 1.0 + imputed = impute_us_scf_education_debt(frame, donor, seed=4, n_estimators=10) + person = frame.table("person") + + single = person["person_household_id"] % 3 == 0 + youngest_child = ~single & (person["A_LINENO"] == 3) + older_child = person["A_LINENO"] == 4 + own = imputed["federal_student_loan_balance"] + plus = imputed["federal_parent_plus_loan_balance"] + # Couples have an adult child, who holds the whole draw as an own loan. + assert (own[youngest_child] > 0).all() + assert (own[~youngest_child & ~single] == 0).all() + assert (own[older_child] == 0).all() + assert ( + imputed.loc[youngest_child, "federal_student_loan_graduate_share"] == 1 + ).all() + assert imputed.loc[youngest_child, "is_income_based_repayment_new_borrower"].all() + # Single parents' children are minors: the child's draw is the head's + # Parent PLUS loan there, and nobody else in the household holds any. + single_head = single & (person["A_LINENO"] == 1) + assert (plus[single_head] > 0).all() + assert (plus[~single_head] == 0).all() + assert (own[single] == 0).all() + + +def test_adult_children_count_children_of_either_partner_aged_18_or_over() -> None: + frame = _us_frame( + pd.DataFrame.from_records( + _household( + [ + {"age": 50.0, "A_SPOUSE": 2, "A_MARITL": 1}, + {"age": 48.0, "A_SPOUSE": 1, "A_MARITL": 1, "is_female": True}, + {"age": 25.0, "PEPAR1": 2}, + {"age": 19.0, "PEPAR1": 1, "PEPAR2": 2}, + {"age": 19.0, "PEPAR1": 1}, + {"age": 12.0, "PEPAR1": 1}, + {"age": 22.0}, + ] + ) + ) + ) + recipients = education_debt._recipients(frame) + + assert recipients.features[ + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR + ].tolist() == [3.0] + # The youngest adult child, the lower line number on a tie. + assert recipients.child_position.tolist() == [3] + + +def test_with_inputs_is_idempotent_and_passes_its_gate() -> None: + frame = _us_frame(_person_rows()) + out = with_us_scf_education_debt_inputs( + frame, + seed=1, + time_period=TIME_PERIOD, + scf_education_debt_donor=_ready_donor(), + n_estimators=15, + ) + again = with_us_scf_education_debt_inputs( + out, + seed=2, + time_period=TIME_PERIOD, + scf_education_debt_donor=_ready_donor(), + n_estimators=15, + ) + assert again is out + gate = us_scf_education_debt_signal_gate(out) + assert gate.passed, gate.failures + summary = us_scf_education_debt_summary(out) + assert summary["household_income_columns_present"] == [ + "employment_income_before_lsr", + "taxable_interest_income", + ] + fit = us_scf_education_debt_fit_table(out) + assert 0 < fit["households"]["share_with_federal_education_debt"] < 1 + assert set(fit["borrowers"]["balance_bands"]) == { + label for label, _, _ in education_debt.SCF_EDUCATION_DEBT_BALANCE_BANDS + } + + +def test_gate_refuses_missing_negative_and_incoherent_columns() -> None: + frame = _us_frame(_person_rows()) + missing = us_scf_education_debt_signal_gate(frame) + assert not missing.passed + assert "person columns missing" in missing.failures[0] + + out = with_us_scf_education_debt_inputs( + frame, + seed=1, + time_period=TIME_PERIOD, + scf_education_debt_donor=_ready_donor(), + n_estimators=15, + ) + tables = {entity: out.table(entity).copy() for entity in out.entities} + person = tables["person"] + person.loc[person.index[0], "federal_student_loan_balance"] = -1.0 + no_balance = person["federal_student_loan_balance"] <= 0 + person.loc[no_balance, "federal_student_loan_graduate_share"] = 0.5 + person["is_income_based_repayment_new_borrower"] = True + broken = Frame( + tables, + out.schema, + {entity: out.weights_for(entity) for entity in out.weighted_entities}, + ) + failures = " ".join(us_scf_education_debt_signal_gate(broken).failures) + assert "negative" in failures + assert "without a federal_student_loan_balance" in failures + assert "without a federal balance" in failures + + +# --------------------------------------------------------------------------- # +# Fit table # +# --------------------------------------------------------------------------- # +def test_fit_table_matches_hand_computed_shares_medians_and_bands() -> None: + table = education_debt_fit_table( + household_debt=np.array([0.0, 4_000.0, 30_000.0, 250_000.0]), + household_weight=np.array([2.0, 1.0, 1.0, 1.0]), + head_age=np.array([70.0, 24.0, 30.0, 40.0]), + race_group=np.array([1, 1, 2, 3]), + income_quintile=np.array([1, 2, 3, 5]), + borrower_balance=np.array([4_000.0, 30_000.0, 250_000.0]), + borrower_age=np.array([24.0, 30.0, 40.0]), + borrower_weight=np.array([1.0, 1.0, 2.0]), + parent_plus_total=25_000.0, + graduate_numerator=10.0, + graduate_denominator=40.0, + new_borrower_weight=1.0, + ) + + households = table["households"] + assert households["share_with_federal_education_debt"] == pytest.approx(3 / 5) + assert households["mean_among_borrowers"] == pytest.approx(284_000 / 3) + assert households["median_among_borrowers"] == 30_000.0 + assert households["parent_plus_share_of_total"] == pytest.approx(25_000 / 284_000) + assert table["household_balance_bands"]["under_5k"] == pytest.approx(1 / 3) + assert table["household_balance_bands"]["200k_plus"] == pytest.approx(1 / 3) + assert ( + table["by_head_age_band"]["62_and_older"]["share_with_federal_education_debt"] + == 0.0 + ) + assert table["by_race"]["white_non_hispanic"][ + "share_with_federal_education_debt" + ] == pytest.approx(1 / 3) + borrowers = table["borrowers"] + assert borrowers["median_balance"] == 30_000.0 + assert borrowers["age_bands"]["24_and_younger"] == pytest.approx(0.25) + assert borrowers["age_bands"]["35_to_49"] == pytest.approx(0.5) + assert borrowers["graduate_share_of_own_original_principal"] == 0.25 + assert borrowers["ibr_new_borrower_share"] == pytest.approx(0.25) + + +def test_donor_fit_table_tabulates_observed_and_imputed_rows_alike() -> None: + donor = _ready_donor(200) + donor["head_federal_student_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["head_own_share"] + ) + donor["spouse_federal_student_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["spouse_own_share"] + ) + donor["child_federal_student_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] * donor["child_own_share"] + ) + donor["federal_parent_plus_loan_balance"] = ( + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] + - donor["head_federal_student_loan_balance"] + - donor["spouse_federal_student_loan_balance"] + - donor["child_federal_student_loan_balance"] + ) + donor["racecl4"] = (np.arange(len(donor)) % 4 + 1).astype(float) + donor["spouse_age"] = donor["age"] + donor["child_borrower_age_code"] = np.where(donor["child_own_share"] > 0, 25.0, 0.0) + donor["youngest_adult_child_age_code"] = donor["child_borrower_age_code"] + observed = scf_education_debt_donor_fit_table(donor) + same = scf_education_debt_donor_fit_table(donor, imputed=donor) + assert json.dumps(observed, sort_keys=True) == json.dumps(same, sort_keys=True) + expected_share = float( + donor.loc[ + donor[SCF_EDUCATION_DEBT_TOTAL_TARGET] > 0, _DONOR_WEIGHT_COLUMN + ].sum() + / donor[_DONOR_WEIGHT_COLUMN].sum() + ) + assert observed["households"]["share_with_federal_education_debt"] == ( + pytest.approx(expected_share) + ) + # Each child borrower counts once, its weight spread over ages 18-27. + children = donor["child_own_share"] > 0 + assert children.any() + head_or_spouse = ( + (donor["head_own_share"] + donor["parent_plus_share"] > 0).astype(float) + + (donor["spouse_own_share"] > 0) + + children + ) + assert observed["borrowers"]["borrower_weight"] == pytest.approx( + float(np.dot(head_or_spouse, donor[_DONOR_WEIGHT_COLUMN])) + ) + + +# --------------------------------------------------------------------------- # +# Recipient definitions # +# --------------------------------------------------------------------------- # +def _household(members: list[dict[str, object]], household_id: int = 1) -> list[dict]: + rows = [] + for index, member in enumerate(members, start=1): + rows.append( + { + "person_id": household_id * 100 + index, + "person_household_id": household_id, + "PH_SEQ": household_id, + "A_LINENO": index, + "A_SPOUSE": 0, + "PECOHAB": 0, + "age": 40.0, + "is_female": False, + "PRDTRACE": 1, + "PRDTHSP": 0, + "A_MARITL": 7, + "PEPAR1": -1, + "PEPAR2": -1, + "employment_income_before_lsr": 0.0, + **member, + } + ) + return rows + + +def _definition_frame() -> Frame: + rows = [ + # 1: a 30-year-old woman householder with her cohabiting partner (31), + # his child, and her 62-year-old mother. + *_household( + [ + {"age": 30.0, "is_female": True, "PECOHAB": 2, + "employment_income_before_lsr": 40_000.0}, + {"age": 31.0, "PECOHAB": 1, "employment_income_before_lsr": 50_000.0}, + {"age": 8.0, "PEPAR1": 2}, + {"age": 62.0, "is_female": True, + "employment_income_before_lsr": 30_000.0}, + ], + household_id=1, + ), + # 2: a married same-sex couple, the householder younger. + *_household( + [ + {"age": 35.0, "A_SPOUSE": 2, "A_MARITL": 1}, + {"age": 45.0, "A_SPOUSE": 1, "A_MARITL": 1}, + ], + household_id=2, + ), + ] # fmt: skip + person = pd.DataFrame.from_records(rows) + return _us_frame(person) + + +def test_recipient_definition_refuses_unknown_rules() -> None: + with pytest.raises(ValueError, match="reference_person"): + education_debt.RecipientDefinition(reference_person="eldest") + with pytest.raises(ValueError, match="income_unit"): + education_debt.RecipientDefinition(income_unit="tax_unit") + + +def test_default_definition_follows_the_scf_definitions() -> None: + frame = _definition_frame() + recipients = education_debt._recipients(frame) + person = frame.table("person") + features = recipients.features + + # The householder's cohabiting partner is the man of the couple; the + # same-sex couple's older spouse leads. + assert person["age"].to_numpy()[recipients.head_position].tolist() == [31.0, 45.0] + assert features["is_married"].tolist() == [1.0, 1.0] + # Household 1 counts the partner's child; the mother is no one's child. + assert features["own_children_in_household"].tolist() == [1.0, 0.0] + # Income sums over the couple and their children; the mother's wages are + # outside the SCF primary economic unit. + assert features["employment_income"].tolist() == [90_000.0, 0.0] + assert list(features.columns) == [ + *education_debt.SCF_EDUCATION_DEBT_CANDIDATE_PREDICTORS, + SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR, + ] + assert features[SCF_EDUCATION_DEBT_ADULT_CHILDREN_PREDICTOR].tolist() == [0.0, 0.0] + + +def test_first_receipt_definition_reproduces_the_oldest_adult_rule() -> None: + frame = _definition_frame() + recipients = education_debt._recipients( + frame, education_debt.SCF_EDUCATION_DEBT_FIRST_RECEIPT_DEFINITION + ) + person = frame.table("person") + + assert person["age"].to_numpy()[recipients.head_position].tolist() == [62.0, 45.0] + assert recipients.features["is_married"].tolist() == [0.0, 1.0] + assert recipients.features["own_children_in_household"].tolist() == [0.0, 0.0] + + +def test_income_units_sum_the_declared_members() -> None: + frame = _definition_frame() + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + # The mother forms her own SPM unit. + spm = tables["person"]["person_spm_unit_id"].to_numpy().copy() + spm[3] = 9_999 + tables["person"]["person_spm_unit_id"] = spm + tables["spm_unit"] = pd.DataFrame({"spm_unit_id": np.unique(spm)}) + frame = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + ) + for unit, expected in ( + ("household", 120_000.0), + ("spm_unit", 90_000.0), + ("reference_couple_and_children", 90_000.0), + ): + definition = education_debt.RecipientDefinition(income_unit=unit) + features = education_debt._recipients(frame, definition).features + assert features["employment_income"].iloc[0] == expected, unit diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 77338f1173..6c2da362e7 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -308,6 +308,9 @@ "release_input_coverage.py", "release_target_parity.py", "scf_auto_loans.py", + # SCF 2025 federal student loan inputs (microcosm#1073); a donor-model + # stage shaped like scf_auto_loans. + "scf_education_debt.py", "scf_wealth.py", "sipp_financial_assets.py", "sipp_tips.py", diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py b/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py index e25bc2619a..c812e70f57 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stored_input_register.py @@ -30,7 +30,8 @@ def test_every_register_entry_is_a_named_producer_column(): base-spine tag of every US entity, the pooled-source provenance columns, microunit's two tax-unit construction columns, the PUF capital-gains tail provenance columns, the ACS-native acs_ amounts and acs_pums's puma_geoid - alias.""" + alias, plus the two pending engine-input families: the parent ids and the + federal student loan inputs.""" support = { column(entity) @@ -61,9 +62,10 @@ def test_every_register_entry_is_a_named_producer_column(): | _ACS_NATIVE_AMOUNTS | {"puma_geoid"} | set(US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS) + | set(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS) ) assert len(_ACS_NATIVE_AMOUNTS) == 6 - assert len(US_STORED_NON_VARIABLE_COLUMNS) == 30 + 6 + 6 + 6 + 1 + 2 + assert len(US_STORED_NON_VARIABLE_COLUMNS) == 30 + 6 + 6 + 6 + 1 + 2 + 4 def test_every_register_reason_names_where_its_column_comes_from(): @@ -96,6 +98,10 @@ def test_every_register_reason_names_where_its_column_comes_from(): column: "US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS" for column in US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS }, + **{ + column: "US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS" + for column in US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS + }, } for column, owner in expected_owner.items(): assert owner in US_STORED_NON_VARIABLE_COLUMNS[column], column diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index ed52466467..1fba52f613 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -343,7 +343,9 @@ def capture_equality(expected: object, actual: object) -> None: # retain_best_objective and microcosm#1301's budget_search_bracket # move it. It covers microcosm.fit.qrf's attested bytes too, so # fit_draw_next's caller-owned uniforms (microcosm#1332) move it. - "spec_sha256": "19460fc168a6f9ed44e580d0e36777cce78bb33cb52b75630ee86203fa795684", + # microcosm#1073's scf_wealth education-debt declaration moves it + # through the bundle's source stages. + "spec_sha256": "eee022d2e258e9e7dfe6b87b80f4b731e3a679486a36abe016db5ec7515dfe10", }, } diff --git a/packages/microcosm-data/src/microcosm/data/stored_inputs.py b/packages/microcosm-data/src/microcosm/data/stored_inputs.py index 5193e0cfcc..936e938554 100644 --- a/packages/microcosm-data/src/microcosm/data/stored_inputs.py +++ b/packages/microcosm-data/src/microcosm/data/stored_inputs.py @@ -284,7 +284,10 @@ #: (microcosm#884). No examined file stores them yet, so they rest on their #: live producer, ``eligibility_inputs.US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS``, #: which ``test_us_stored_input_register.py`` binds, and leave the register -#: once the pinned engine defines them. +#: once the pinned engine defines them. The four federal student loan inputs +#: (microcosm#1073) likewise postdate them: their producer is +#: ``scf_education_debt.US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS``, and they leave +#: the register once the pinned engine carries policyengine-us#9721 and #9724. #: #: Two model-named non-variable columns that the published default, its #: receipt child and the ACS local-area release store are deliberately absent, @@ -447,6 +450,42 @@ ("parent_2_id", "PEPAR2"), ) }, + **{ + column: ( + f"{detail}, imputed from the 2025 Survey of Consumer Finances by " + "microcosm.build.us_runtime.scf_education_debt " + "(US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS; microcosm#1073), and " + "written only when the fiscal-refresh build runs with " + "--scf-education-debt. The certified engine does not define it " + f"yet, so it is ignored there; policyengine-us#{pull} adds it as " + "an input. Remove this entry when the pinned engine defines the " + "variable." + ) + for column, detail, pull in ( + ( + "federal_student_loan_balance", + "The person's own federal student loan balance", + 9721, + ), + ( + "federal_parent_plus_loan_balance", + "The person's federal Parent PLUS loan balance", + 9721, + ), + ( + "is_income_based_repayment_new_borrower", + "Whether every federal loan the person holds was borrowed in " + "2014 or later (IBR new-borrower status)", + 9724, + ), + ( + "federal_student_loan_graduate_share", + "The graduate or professional share of the original principal " + "of the person's own federal student loans", + 9724, + ), + ) + }, } ) diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 1d051f5c4d..0b20fdbc36 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -123,7 +123,7 @@ def binding() -> dict[str, object]: "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -133,7 +133,7 @@ def binding() -> dict[str, object]: "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 938, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index ef4bc8868b..03b82ddc95 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/test_support/microcosm_build/us_stored_input_register.py b/test_support/microcosm_build/us_stored_input_register.py index 91c008846b..f191a253e4 100644 --- a/test_support/microcosm_build/us_stored_input_register.py +++ b/test_support/microcosm_build/us_stored_input_register.py @@ -31,6 +31,9 @@ US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS, ) from microcosm.build.us_runtime.operator_boundary import _ACS_NATIVE_INPUT_CONTRACTS +from microcosm.build.us_runtime.scf_education_debt import ( + US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, +) from microcosm.build.us_runtime.support_provenance import ( spine_source_id_column, support_channel_column, diff --git a/test_support/microcosm_data/stored_inputs.py b/test_support/microcosm_data/stored_inputs.py index 4206bdf943..d42aed8769 100644 --- a/test_support/microcosm_data/stored_inputs.py +++ b/test_support/microcosm_data/stored_inputs.py @@ -155,9 +155,20 @@ def _column_strategies(st): #: Register entries that postdate the files examined for #1026 and rest on a #: live producer instead: the parent ids (microcosm#884), bound to -#: eligibility_inputs.US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS by +#: eligibility_inputs.US_ELIGIBILITY_INPUTS_PARENT_ID_COLUMNS, and the four +#: federal student loan inputs (microcosm#1073), bound to +#: scf_education_debt.US_SCF_EDUCATION_DEBT_OUTPUT_COLUMNS, both by #: test_us_stored_input_register.py (microcosm-data cannot import the build). -_PRODUCER_BOUND_ENTRIES = frozenset({"parent_1_id", "parent_2_id"}) +_PRODUCER_BOUND_ENTRIES = frozenset( + { + "parent_1_id", + "parent_2_id", + "federal_student_loan_balance", + "federal_parent_plus_loan_balance", + "is_income_based_repayment_new_borrower", + "federal_student_loan_graduate_share", + } +) #: The role columns policyengine-core's ``build_from_dataset`` reads to build diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 44bebf56c4..3dd560c3c0 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -84,6 +84,8 @@ CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR, CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE, ORG_2024_DONOR_CONTENT_SHA256, + SCF_2025_FULL_EXTRACT, + SCF_2025_SUMMARY_EXTRACT, SIPP_2023_FINANCIAL_ASSET_DONOR_SHA256, SIPP_2023_FINANCIAL_ASSET_DONOR_SIZE_BYTES, SIPP_2023_HEAD_START_DONOR_SHA256, @@ -114,6 +116,7 @@ fetch_org_2024_donor, fetch_scf_2022_full_extract, fetch_scf_2022_summary_extract, + fetch_scf_public_file, fetch_sipp_2023_financial_asset_donor, fetch_sipp_2023_tip_donor, hard_target_package_aliases, @@ -122,6 +125,7 @@ load_org_2024_donor, load_scf_2022_auto_loan_donor, load_scf_2022_financial_asset_donor, + load_scf_2025_education_debt_donor, load_sipp_2023_financial_asset_donor, load_sipp_2023_head_start_donor, load_sipp_2023_ssi_disability_donor, @@ -167,6 +171,8 @@ us_retirement_distributions_signal_gate, us_salt_refund_income_signal_gate, us_scf_auto_loans_signal_gate, + us_scf_education_debt_fit_table, + us_scf_education_debt_signal_gate, us_scf_wealth_signal_gate, us_sipp_head_start_signal_gate, us_sipp_tips_signal_gate, @@ -205,6 +211,7 @@ with_us_retirement_contribution_inputs, with_us_retirement_distribution_inputs, with_us_scf_auto_loan_inputs, + with_us_scf_education_debt_inputs, with_us_scf_wealth_inputs, with_us_sipp_head_start_input, with_us_sipp_tip_inputs, @@ -1602,6 +1609,35 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "interest. When omitted, scf2022s.zip is fetched and cached." ), ) + parser.add_argument( + "--scf-education-debt", + action="store_true", + help=( + "Impute the four federal student loan person inputs " + "(federal_student_loan_balance, federal_parent_plus_loan_balance, " + "is_income_based_repayment_new_borrower, " + "federal_student_loan_graduate_share) from the 2025 SCF " + "(microcosm#1073). Off by default until the pinned policyengine-us " + "defines them (policyengine-us#9721, #9724); the certified engine " + "ignores them until then." + ), + ) + parser.add_argument( + "--scf-2025-summary-extract", + type=Path, + help=( + "Optional local rscfp2025.dta for --scf-education-debt. When " + "omitted, the sha-pinned scfp2025s.zip is fetched and cached." + ), + ) + parser.add_argument( + "--scf-2025-full-extract", + type=Path, + help=( + "Optional local p25i6.dta for --scf-education-debt. When omitted, " + "the sha-pinned scf2025s.zip is fetched and cached." + ), + ) parser.add_argument( "--sipp-tip-donor", type=Path, @@ -12485,6 +12521,78 @@ def _check_committed_us_ledger_feed_pin( ) +def _with_scf_education_debt_stage( + base_frame: Frame, + *, + args: argparse.Namespace, + telemetry: Any, + fit_path: Path | None, +) -> Frame: + """Run the opt-in SCF 2025 education-debt stage and its signal gate.""" + + if telemetry is not None: + telemetry.stage( + "scf_education_debt_inputs", + message=( + "Imputing federal student loan balances, Parent PLUS balances, " + "IBR new-borrower status and graduate shares from the 2025 SCF." + ), + ) + summary_path = ( + Path(args.scf_2025_summary_extract) + if args.scf_2025_summary_extract is not None + else fetch_scf_public_file(SCF_2025_SUMMARY_EXTRACT) + ) + full_path = ( + Path(args.scf_2025_full_extract) + if args.scf_2025_full_extract is not None + else fetch_scf_public_file(SCF_2025_FULL_EXTRACT) + ) + donor = load_scf_2025_education_debt_donor(summary_path, full_path) + base_frame = with_us_scf_education_debt_inputs( + base_frame, + seed=args.seed, + time_period=PERIOD, + scf_education_debt_donor=donor, + ) + gate = us_scf_education_debt_signal_gate(base_frame) + if not gate.passed: + if telemetry is not None: + telemetry.stage( + "scf_education_debt_gate", + status="failed", + message="SCF education-debt signal gate failed.", + failures=list(gate.failures), + force_upload=True, + ) + raise RuntimeError( + "Release gates failed: " + + "; ".join( + f"SCF education-debt signal failed: {failure}" + for failure in gate.failures + ) + ) + if fit_path is not None: + # Design-weight fit of the staged frame against the SCF; the receipt + # tool tabulates the calibrated export the same way (microcosm#1073). + fit_path.parent.mkdir(parents=True, exist_ok=True) + fit_path.write_text( + json.dumps( + { + "schema_version": 1, + "weights": "pre_calibration_household_weights", + "fit": us_scf_education_debt_fit_table(base_frame), + }, + indent=2, + sort_keys=True, + default=float, + ) + + "\n", + encoding="utf-8", + ) + return base_frame + + def _main(argv: Sequence[str] | None = None) -> int | None: global _ACTIVE_EMITTER, _ACTIVE_PROGRESS _ACTIVE_EMITTER = None @@ -14067,6 +14175,21 @@ def _main(argv: Sequence[str] | None = None) -> int | None: for failure in scf_auto_loan_gate.failures ) ) + if args.scf_education_debt: + # microcosm#1073: federal student loan balances from the 2025 SCF. The + # pinned engine does not define these inputs yet, so the stage is + # opt-in and its columns ride in the stored-input register until it + # does (microcosm.data.stored_inputs). + base_frame = _with_scf_education_debt_stage( + base_frame, + args=args, + telemetry=telemetry, + fit_path=( + artifact_root / "us_scf_education_debt_fit.json" + if dry_run is None + else None + ), + ) if telemetry is not None: telemetry.stage( "sipp_vehicle_inputs", diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 5dc45835db..376cfe85de 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "5588726fa6dc5e216faff42392e0e39d75f1a58b1ea14921f36abdffafbcb36b" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index c066d573ce..51b06f3156 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7" + "f26245d12f4cd77a77ab4b595c9351f6923330815a87b5734e9c80e2a0a129a9" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/core.py b/tools/us_bundle_generation/core.py index aa6448f100..38c38d9aa6 100644 --- a/tools/us_bundle_generation/core.py +++ b/tools/us_bundle_generation/core.py @@ -911,6 +911,16 @@ def build_vintages() -> dict[str, Any]: "field": "vintage", }, }, + { + "id": "scf_2025", + "kind": "survey_period_ref", + "authority_ref": { + "kind": "source_stage_artifact", + "stage": "scf_wealth", + "artifact_index": 3, + "field": "vintage", + }, + }, { "id": "sipp_2023", "kind": "survey_period_ref", @@ -958,6 +968,7 @@ def build_vintages() -> dict[str, Any]: "puma_2020": {"cd_119", "census_2020"}, "release_us_2024": {"target_2024"}, "scf_2022": {"target_2024"}, + "scf_2025": {"target_2024"}, "sipp_2023": {"target_2024"}, "target_2024": { "acs_2022", @@ -969,6 +980,7 @@ def build_vintages() -> dict[str, Any]: "policyengine_us_surface", "release_us_2024", "scf_2022", + "scf_2025", "sipp_2023", "tax_2015", },