diff --git a/docs/design/track_b_holdout_registry.json b/docs/design/track_b_holdout_registry.json new file mode 100644 index 00000000..bf4c7024 --- /dev/null +++ b/docs/design/track_b_holdout_registry.json @@ -0,0 +1,373 @@ +{ + "schema": "track_b_holdout_registry.v1", + "title": "Held-out data registry for Track B and the open-population lane", + "purpose": "The single registry of the data that every Track B gate and every open-population gate holds out, set up by the dynamics coordinator on 2026-10-21. It is part of Track B's B0.2 exposure record (design revision 3, section 5.2, elements 4 and 5). Cite it by path, commit and SHA-256. src/populace_dynamics/track_b/holdout_registry.py states and enforces its rules, and tests/test_track_b_holdout_registry.py runs them on every commit.", + "rules": [ + "A gate registers what it holds out here, as a lone candidate entry in a commit of its own, before its record freezes and before its scored run is committed. It moves to frozen, used or withdrawn only in later commits.", + "J's candidate holdouts are registered first, because J is on Track B's critical path. OP0.2 registers its picks for OP12's gate_ssa_totals here before it freezes them, and picks from the years J has not taken.", + "Every unit key names a source from the controlled vocabulary in src/populace_dynamics/track_b/holdout_registry.py (SOURCES), then its years (a year, an inclusive range, or 'undated'), then optional scope segments. The registry's sources must equal that vocabulary. Each source declares a family, a title, a catalog reference no other source shares, and what its key year means. Adding a source is a reviewed code change, and a new source must not be another name for a declared one.", + "Two unit keys share a unit when their source is the same and their years intersect; scope segments describe the unit and never narrow the overlap. Live entries that share a unit, or whose relation is unresolved, list each other in disclosed_overlaps, and both gates' registrations name the overlap. No unit is held out by two gates silently.", + "The overlap rule cannot see two sources that share underlying data, or two editions of a publication that report the same data years. So an entry reviews every earlier live entry of the same source family that it neither shares a unit with nor discloses. A review records the outcome 'disjoint', a substantive reason and citations (path:line or https URLs); an unresolved relation is disclosed instead.", + "Reference years are recorded separately from interview or publication availability (design section 5.2, element 10): a unit key's year is what its source declares (for PSID, the interview wave; for EPUF, the file's vintage year), and each entry's units state the reference or event years.", + "An entry whose gate has scored has status 'used'. Its scored_evidence cites scored-run artifacts listed in the module's SCORED_RUNS, which pins each artifact's gate and the SHA-256 of its bytes; an artifact joins that list, as a reviewed code change, when its gate scores. Each citation also names an identity field (gate, gate.name or registration_id) whose value identifies the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. A registration's scored-run bytes must not have been committed, at any path and whether or not they were later moved or deleted, in the history of the commit that first records the entry on the base branch's first-parent chain. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", + "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused. These rules hold against the base commit, between every pair of consecutive first-parent commits on the base branch (each snapshot also validated against its own commit's tree), and between consecutive commits of a pull request's branch. A newly created registry is admitted against an empty one. A pull request that creates or changes the registry is merged with a merge commit, never rebased, so its draft commits do not enter the base branch's first-parent history.", + "A historical import records a gate that scored before this registry existed. It starts as used, and its id, gate, unit keys and scored runs must match an authorization in the module's HISTORICAL_IMPORTS. B0.2's mechanical inventory adds an import it finds, or a correction to an existing one, as a new authorized entry in a reviewed code change. Imports are allowed only while the registry is incomplete.", + "The registry is incomplete until completion_record, B0.2's mechanical exposure inventory, exists. It starts with the prior uses verified below; the inventory adds the rest before any B0.2 gate record freezes." + ], + "status_values": { + "candidate": "registered by a gate's B0.2 or OP0.2 record; not yet frozen", + "frozen": "fixed in a frozen gate record before that gate's candidates run", + "used": "scored by a registered gate; exposure for every later gate", + "withdrawn": "released before use, with the reason recorded" + }, + "sources": { + "psid": { + "family": "psid", + "title": "Panel Study of Income Dynamics: family files, cross-year individual file and supplements", + "catalog": "University of Michigan Institute for Social Research: Panel Study of Income Dynamics", + "key_year": "interview wave (collection year); income reference years are the wave year minus one" + }, + "ssa_epuf": { + "family": "ssa_administrative", + "title": "SSA Earnings Public-Use File", + "catalog": "SSA Office of Research, Evaluation, and Statistics: Earnings Public-Use File", + "key_year": "the vintage year in the file's title: the 2006 file holds earnings through 2006 and was released in August 2011" + }, + "ssa_supplement": { + "family": "ssa_administrative", + "title": "Annual Statistical Supplement to the Social Security Bulletin", + "catalog": "SSA Office of Research, Evaluation, and Statistics: Annual Statistical Supplement to the Social Security Bulletin", + "key_year": "edition year" + }, + "ssa_di_asr": { + "family": "ssa_administrative", + "title": "Annual Statistical Report on the Social Security Disability Insurance Program", + "catalog": "SSA Office of Research, Evaluation, and Statistics: Annual Statistical Report on the Social Security Disability Insurance Program", + "key_year": "edition year" + }, + "census_cps_tables": { + "family": "us_cross_section", + "title": "U.S. Census Bureau Current Population Survey tables", + "catalog": "U.S. Census Bureau: Current Population Survey published tables", + "key_year": "table reference year" + }, + "populace_us_frame": { + "family": "us_cross_section", + "title": "Microcosm US certified frame", + "catalog": "hf://policyengine/populace-us/populace_us_2024.h5", + "key_year": "frame vintage" + }, + "mermin_smith_anchors": { + "family": "literature", + "title": "Mermin and Smith published orderings, committed as anchors", + "catalog": "gates.yaml gate_w1 holdout_basis: mermin_smith_committed_orderings", + "key_year": "undated" + } + }, + "complete": false, + "completion_record": "docs/design/track_b_b0_2_exposure_inventory.json", + "entries": [ + { + "id": "H001", + "provenance": "historical_import", + "gate": "gate_m6", + "gate_identifiers": [ + "gate_m6", + "m6_temporal_holdout_projection_drift" + ], + "lane": "M6 (locked gate)", + "status": "used", + "source": "PSID family and cross-year individual files", + "units": { + "boundary_T_star": 2014, + "seed_wave": 2015, + "held_out_window": "2015-2022", + "gated_flow_event_years": [ + 2015, + 2016, + 2017, + 2018, + 2019 + ], + "gated_earnings_reference_years": [ + 2016, + 2018 + ], + "report_only_shock_earnings_reference_years": [ + 2020, + 2022 + ], + "report_only_shock_flow_event_years": [ + 2020, + 2021 + ], + "interview_waves": "2015 (seed) to 2023" + }, + "unit_keys": [ + "psid:2015-2023:held_out_window" + ], + "evidence": [ + "gates.yaml:5374-5376 (fit <= T*=2014; score the held-out PSID window 2015-2022)", + "gates.yaml:5496-5508 (fit_holdout and shock_window)" + ], + "scored_evidence": [ + { + "path": "runs/gate_m6_candidate1_v1.json", + "field": "gate.name", + "equals": "gate_m6" + }, + { + "path": "runs/gate_m6_candidate2_v1.json", + "field": "gate.name", + "equals": "gate_m6" + }, + { + "path": "runs/gate_m6_candidate3_v1.json", + "field": "gate.name", + "equals": "gate_m6" + } + ], + "disclosed_overlaps": [ + "H002", + "H004" + ], + "overlap_review": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; shares PSID interview wave 2015 with B2 (H002) and waves 2015-2023 with gate_m4's panel (H004)" + } + ] + }, + { + "id": "H002", + "provenance": "historical_import", + "gate": "B2", + "gate_identifiers": [ + "B2", + "6252388925" + ], + "lane": "Track B", + "status": "used", + "source": "PSID family files FAM2013ER and FAM2015ER, with the cross-year individual file", + "units": { + "reference_years": [ + 2012, + 2014 + ], + "interview_waves": [ + 2013, + 2015 + ], + "anchor_wave_not_held_out": 2011, + "ages": "25-64 at the collection wave", + "population": "2011 full-anchor persons with a valid 2010 row" + }, + "unit_keys": [ + "psid:2013:fam2013er", + "psid:2015:fam2015er" + ], + "evidence": [ + "src/populace_dynamics/track_b/b2.py:64-65", + "docs/design/track_b_b0_1_audit.md, section 16 (definitions)", + "registration: issue #42 comment 6252388925" + ], + "scored_evidence": [ + { + "path": "runs/track_b_b2_score.json", + "field": "registration_id", + "equals": "6252388925" + } + ], + "disclosed_overlaps": [ + "H001", + "H004" + ], + "overlap_review": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "B2 scored on 2026-10-18; seeded at 1e739d32; its 2015 wave is gate_m6's seed wave (H001), and both waves fall in gate_m4's panel (H004)" + } + ] + }, + { + "id": "H003", + "provenance": "historical_import", + "gate": "gate_epuf_fill", + "gate_identifiers": [ + "gate_epuf_fill", + "epuf_career_fill", + "2026-10-03-epuf-career-fill" + ], + "lane": "EPUF career fill (locked gate)", + "status": "used", + "source": "SSA 2006 Earnings Public-Use File (EPUF)", + "units": { + "partition": "EPUF2006_TEST_by_salted_person_hash", + "earnings_years": "1951-2006" + }, + "unit_keys": [ + "ssa_epuf:2006:test_partition" + ], + "evidence": [ + "gates.yaml:5918 (holdout_basis)", + "src/populace_dynamics/data/epuf.py:15 (capped taxable earnings, 1951-2006)" + ], + "scored_evidence": [ + { + "path": "runs/epuf_fill_gate_test_v1.json", + "field": "registration_id", + "equals": "2026-10-03-epuf-career-fill" + } + ], + "disclosed_overlaps": [ + "H004", + "H005" + ], + "overlap_review": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; its relation to the DI ASR 2023 (H004, H005) is unresolved and disclosed; the open-population design (PR #557, draft) plans gate_young_earn on a new salt of the same file, which shares this unit and must disclose it" + } + ] + }, + { + "id": "H004", + "provenance": "historical_import", + "gate": "gate_m4", + "gate_identifiers": [ + "gate_m4", + "m4_disability" + ], + "lane": "M4 disability (locked gate)", + "status": "used", + "source": "PSID cross-year individual file employment status; SSA Annual Statistical Report on the SSDI Program, 2023", + "units": { + "holdout": "person-disjoint side-A half of each of 100 split seeds over the work-limitation panel", + "employment_status_waves": "1982, 1983, 1993-1997 annual, 1999-2023 biennial", + "di_asr_2023_tables": [ + 19, + 35, + 36, + 49, + 50 + ] + }, + "unit_keys": [ + "psid:1982:employment_status", + "psid:1983:employment_status", + "psid:1993-1997:employment_status", + "psid:1999-2023:employment_status", + "ssa_di_asr:2023" + ], + "evidence": [ + "gates.yaml:2945 (holdout_basis)", + "gates.yaml:2946-2953 (data_staged)", + "src/populace_dynamics/data/disability.py:19-24 (wave coverage)" + ], + "scored_evidence": [ + { + "path": "runs/gate_m4_hazard_v1.json", + "field": "gate", + "equals": "gate_m4" + } + ], + "disclosed_overlaps": [ + "H001", + "H002", + "H003", + "H005" + ], + "overlap_review": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; its PSID panel shares waves with gate_m6 (H001) and B2 (H002), and the DI ASR 2023 edition with gate_w1 (H005); whether a DI ASR 2023 table reports data years that EPUF 2006 also covers (H003) is unresolved, so it is disclosed" + } + ] + }, + { + "id": "H005", + "provenance": "historical_import", + "gate": "gate_w1", + "gate_identifiers": [ + "gate_w1", + "w1_representative_frame_transport" + ], + "lane": "W1 frame transport (locked gate)", + "status": "used", + "source": "SSA Annual Statistical Supplement 2023, section 6.B; SSA Annual Statistical Report on the SSDI Program, 2023; Census CPS tables; the certified 2024 frame; Mermin and Smith orderings", + "units": { + "holdout_basis": [ + "populace_us_2024_certified", + "ssa_supplement_6b", + "di_asr_2023", + "mermin_smith_committed_orderings" + ], + "tables_named": [ + "Supplement 2023 table 6.B5.1", + "Supplement 2023 tables 6.B4 and 6.B3", + "DI ASR 2023 table 19", + "Census HH-4, reference year 2023", + "Census AD-3, reference year 2023" + ], + "note": "whether each named table was gated or report-only is itemized by B0.2's inventory" + }, + "unit_keys": [ + "ssa_supplement:2023:6.b5.1", + "ssa_supplement:2023:6.b4", + "ssa_supplement:2023:6.b3", + "ssa_di_asr:2023:table_19", + "census_cps_tables:2023:hh-4", + "census_cps_tables:2023:ad-3", + "populace_us_frame:2024:certified", + "mermin_smith_anchors:undated" + ], + "evidence": [ + "gates.yaml:3735-3736 (holdout_basis)", + "gates.yaml:4582-4584 (sources)", + "gates.yaml:4311-4327 (the Census HH-4 and AD-3 2023 sources, recorded as report-only)" + ], + "scored_evidence": [ + { + "path": "runs/gate_w1_candidate1_v1.json", + "field": "gate", + "equals": "gate_w1" + }, + { + "path": "runs/gate_w1_candidate2_v1.json", + "field": "gate", + "equals": "gate_w1" + }, + { + "path": "runs/gate_w1_candidate3_v1.json", + "field": "gate", + "equals": "gate_w1" + }, + { + "path": "runs/gate_w1_candidate4_v1.json", + "field": "gate", + "equals": "gate_w1" + } + ], + "disclosed_overlaps": [ + "H003", + "H004" + ], + "overlap_review": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; shares the DI ASR 2023 edition with gate_m4 (H004); whether its 2023 Supplement and DI ASR tables report data years that EPUF 2006 also covers (H003) is unresolved, so it is disclosed" + } + ] + } + ] +} diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json new file mode 100644 index 00000000..7e011ba9 --- /dev/null +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -0,0 +1,857 @@ +{ + "schema_version": 1, + "verification_class": "accounting conformance", + "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", + "mutation_method": "One deliberate production-line change per check; both registry test modules run in full and are expected to fail; original bytes restored before the next check. No mutated implementation is committed.", + "restored_source_sha256": { + "src/populace_dynamics/track_b/holdout_registry.py": "a1fd1d999f59abecb6a8ca08801a7b2bbf7c4960cf3581f8c77e6cf2972ee354" + }, + "checks": [ + { + "mutation": "scope segments unchecked", + "old": "if not all(SCOPE_RX.fullmatch(part) for part in parts[2:]):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2020::x]", + "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2020:Upper]" + ] + }, + { + "mutation": "reversed year range accepted", + "old": "if end < start:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2022-2020]" + ] + }, + { + "mutation": "different sources overlap", + "old": "if left is None or right is None or left[0] != right[0]:", + "new": "if left is None or right is None:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen", + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", + "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused", + "tests/test_track_b_holdout_registry.py::test_appending_lone_candidates_keeps_every_rule", + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", + "tests/test_track_b_holdout_registry.py::test_settled_registries_are_valid", + "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", + "tests/test_track_b_holdout_registry_committed.py::test_the_committed_registry_keeps_its_rules" + ] + }, + { + "mutation": "undated overlaps nothing", + "old": "if left[1] is None or right[1] is None:\n return True", + "new": "if left[1] is None or right[1] is None:\n return False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", + "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule" + ] + }, + { + "mutation": "ranges overlap only when equal", + "old": "return left[1][0] <= right[1][1] and right[1][0] <= left[1][1]", + "new": "return left[1] == right[1]", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", + "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", + "tests/test_track_b_holdout_registry_committed.py::test_the_seeded_disclosures_are_as_specified" + ] + }, + { + "mutation": "paths may climb out", + "old": "and \"..\" not in path.split(\"/\")", + "new": "and True", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_committed_record" + ] + }, + { + "mutation": "symlinks read as committed files", + "old": "and fields[0] in (\"100644\", \"100755\")", + "new": "and fields[0] in (\"100644\", \"100755\", \"120000\")", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused" + ] + }, + { + "mutation": "any id may be imported", + "old": "return [f\"{name}: is not an authorized historical import\"]", + "new": "return []", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_authorization_admits_one_import_only", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_rename-not an authorized historical import]" + ] + }, + { + "mutation": "imports need no authorization", + "old": "problems.extend(_import_problems(entry, imports))", + "new": "pass", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_authorization_admits_one_import_only", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_runs-differs from its authorized runs]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_units-unit keys differ]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_other_units-unit keys differ]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_regate-is authorized for 'gate_0']", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_rename-not an authorized historical import]" + ] + }, + { + "mutation": "imports may change gate", + "old": "if entry[\"gate\"] != authorized[\"gate\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_regate-is authorized for 'gate_0']" + ] + }, + { + "mutation": "imports may change units", + "old": "if sorted(entry[\"unit_keys\"]) != sorted(authorized[\"unit_keys\"]):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_units-unit keys differ]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_other_units-unit keys differ]" + ] + }, + { + "mutation": "imports may cite other runs", + "old": "if cited != sorted(authorized[\"scored_runs\"]):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_runs-differs from its authorized runs]" + ] + }, + { + "mutation": "unknown gates accepted", + "old": "if not authoritative:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_identifiers_must_be_authoritative" + ] + }, + { + "mutation": "self-declared identifiers accepted", + "old": "allowed &= authoritative", + "new": "pass", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_identifiers_must_be_authoritative" + ] + }, + { + "mutation": "unregistered registrations may cite scores", + "old": "if registered is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_used_registration_must_already_be_registered" + ] + }, + { + "mutation": "any runs file counts as scored", + "old": "if run is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses3-used-not a legal]" + ] + }, + { + "mutation": "another gate's run accepted", + "old": "if run[0] != entry[\"gate\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change4-'gate_1''s scored run]" + ] + }, + { + "mutation": "any field may identify", + "old": "if field not in IDENTITY_FIELDS:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change1-not an identity field]" + ] + }, + { + "mutation": "evidence may name another gate", + "old": "if equals not in allowed:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_identifiers_must_be_authoritative", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change0-does not identify this gate]" + ] + }, + { + "mutation": "pinned bytes unchecked", + "old": "if hashlib.sha256(content).hexdigest() != run[1]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_scored_run_must_keep_its_pinned_bytes" + ] + }, + { + "mutation": "evidence need not identify the gate", + "old": "if lookup(document, field) != equals:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_is_read_from_the_committed_tree" + ] + }, + { + "mutation": "registration may follow its score", + "old": "elif earlier:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]", + "tests/test_track_b_holdout_registry.py::test_a_replacement_ref_cannot_hide_a_committed_score", + "tests/test_track_b_holdout_registry.py::test_a_score_added_only_in_a_merge_resolution_is_refused", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[deleted_and_restored]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[moved]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[same_commit]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]", + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed" + ] + }, + { + "mutation": "an unanswerable timing check passes", + "old": "if earlier is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure0]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure1]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure2]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure3]", + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed", + "tests/test_track_b_holdout_registry.py::test_an_unanswerable_timing_check_fails_closed" + ] + }, + { + "mutation": "scores never found in history", + "old": "if found.stdout.strip():\n return True", + "new": "if False:\n return True", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]", + "tests/test_track_b_holdout_registry.py::test_a_replacement_ref_cannot_hide_a_committed_score", + "tests/test_track_b_holdout_registry.py::test_a_score_added_only_in_a_merge_resolution_is_refused", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[deleted_and_restored]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[moved]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[same_commit]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]", + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed" + ] + }, + { + "mutation": "registration order ignores the first parent", + "old": "\"log\",\n \"--first-parent\",", + "new": "\"log\",\n \"--topo-order\",", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]" + ] + }, + { + "mutation": "entries without ids reach diagnostics", + "old": "isinstance(entry, Mapping) and isinstance(entry.get(\"id\"), str)", + "new": "isinstance(entry, Mapping)", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "JSON parser limits raise", + "old": "except (ValueError, RecursionError):\n return None", + "new": "except json.JSONDecodeError:\n return None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "YAML parser limits raise", + "old": "except (yaml.YAMLError, ValueError, RecursionError):", + "new": "except yaml.YAMLError:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "NUL arguments raise", + "old": "except (ValueError, TypeError, OSError) as error:\n return subprocess.CompletedProcess(", + "new": "except (TypeError, OSError) as error:\n return subprocess.CompletedProcess(", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "any review outcome", + "old": "if item[\"outcome\"] not in REVIEW_OUTCOMES:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change0-is disclosed]" + ] + }, + { + "mutation": "placeholder reasons accepted", + "old": "if len(reason) < MIN_REASON or PLACEHOLDER_RX.search(reason):", + "new": "if len(reason) < MIN_REASON:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change1-substantive]" + ] + }, + { + "mutation": "reviews need no citations", + "old": "if not isinstance(citations, list) or not citations:", + "new": "if not isinstance(citations, list):", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change3-cites nothing]" + ] + }, + { + "mutation": "citations need no form", + "old": "if match is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change4-not path:line or a URL]", + "tests/test_track_b_holdout_registry.py::test_review_citations_must_name_committed_text_lines" + ] + }, + { + "mutation": "citation lines unbounded", + "old": "if end > len(text.splitlines()):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_review_citations_must_name_committed_text_lines" + ] + }, + { + "mutation": "completion record path unchecked", + "old": "if not safe_path(record):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_committed_record" + ] + }, + { + "mutation": "keys may name undeclared sources", + "old": "elif parsed[0] not in sources:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_key_naming_no_source_is_refused" + ] + }, + { + "mutation": "any first status", + "old": "if history[0][\"status\"] not in first:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses1-candidate-starts as one of]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses2-frozen-starts as one of]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses0-frozen-starts as one of]" + ] + }, + { + "mutation": "dates not parsed", + "old": "date = dt.date.fromisoformat(item[\"date\"])", + "new": "date = previous_date", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-10-01]", + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-99-99]", + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[yesterday]" + ] + }, + { + "mutation": "dates may go back", + "old": "if previous_date is not None and date < previous_date:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-10-01]" + ] + }, + { + "mutation": "any transition", + "old": "status not in TRANSITIONS[previous_status]", + "new": "False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses4-frozen-not a legal]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses3-used-not a legal]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses0]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses1]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses2]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses3]" + ] + }, + { + "mutation": "status need not match history", + "old": "if entry[\"status\"] != previous_status:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses5-candidate-last status]" + ] + }, + { + "mutation": "used needs no scored evidence", + "old": "if entry[\"status\"] == \"used\" and not entry[\"scored_evidence\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change0-does not identify this gate]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change1-not an identity field]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change4-'gate_1''s scored run]" + ] + }, + { + "mutation": "alias sources accepted", + "old": "if catalog in catalogs:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_the_registry_must_carry_the_controlled_vocabulary" + ] + }, + { + "mutation": "complete needs no record", + "old": "and committed.blob(record) is None", + "new": "and False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_committed_record" + ] + }, + { + "mutation": "one identifier, two gates", + "old": "if owner != entry[\"gate\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_repeated_ids_and_shared_gate_identifiers_are_refused" + ] + }, + { + "mutation": "one-way disclosure accepted", + "old": "elif entry[\"id\"] not in by_id[other][\"disclosed_overlaps\"]:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_one_way_disclosure_is_refused" + ] + }, + { + "mutation": "withdrawn entries counted live", + "old": "live = [entry for entry in entries if entry[\"status\"] != \"withdrawn\"]", + "new": "live = list(entries)", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen", + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", + "tests/test_track_b_holdout_registry.py::test_appending_lone_candidates_keeps_every_rule", + "tests/test_track_b_holdout_registry.py::test_settled_registries_are_valid" + ] + }, + { + "mutation": "silent shared unit accepted", + "old": "other[\"id\"] not in entry[\"disclosed_overlaps\"]\n ):", + "new": "False\n ):", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused" + ] + }, + { + "mutation": "reviews may name later entries", + "old": "if position.get(name, index) >= index:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_must_name_an_earlier_entry" + ] + }, + { + "mutation": "family relations need no review", + "old": "and other[\"id\"] not in reviewed", + "new": "and False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused", + "tests/test_track_b_holdout_registry.py::test_fixed_family_cases_follow_the_rule" + ] + }, + { + "mutation": "rules may shrink", + "old": "if not _extends(base.get(\"rules\", []), current.get(\"rules\", [])):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" + ] + }, + { + "mutation": "sources may change", + "old": "if sources.get(source) != declared:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" + ] + }, + { + "mutation": "complete may revert", + "old": "if base.get(\"complete\") is True and current.get(\"complete\") is not True:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" + ] + }, + { + "mutation": "entries may be removed or reordered", + "old": "if new_ids[: len(old_ids)] != old_ids:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_any_json_in_any_field_is_reported_not_raised", + "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused", + "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" + ] + }, + { + "mutation": "immutable fields may change", + "old": "if after.get(key) != before.get(key):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-edit_then_restore]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-edit_then_restore]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]", + "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused", + "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" + ] + }, + { + "mutation": "extending lists may shrink", + "old": "if not _extends(before.get(key, []), after.get(key, [])):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_extending_lists_never_shrink" + ] + }, + { + "mutation": "imports allowed after completion", + "old": "if base.get(\"complete\") is True:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_close_when_the_registry_is_complete" + ] + }, + { + "mutation": "registrations may arrive frozen", + "old": "elif statuses != [\"candidate\"]:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_new_registry_is_admitted_against_an_empty_one", + "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen" + ] + }, + { + "mutation": "push ignores its before commit", + "old": "if set(before) != {\"0\"}:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"}]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]", + "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" + ] + }, + { + "mutation": "base branch compares with itself", + "old": "if base == head:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_no_base_raises_rather_than_passing", + "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" + ] + }, + { + "mutation": "absent base registry raises", + "old": "if not listed.stdout.strip():\n return None", + "new": "if False:\n return None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_clean_history_has_no_problems", + "tests/test_track_b_holdout_registry.py::test_a_pull_request_compares_with_its_merge_base", + "tests/test_track_b_holdout_registry_committed.py::test_the_committed_registry_only_appends_to_its_base" + ] + }, + { + "mutation": "snapshots not validated in history", + "old": "if full:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_each_snapshot_is_validated_against_its_own_tree" + ] + }, + { + "mutation": "merge diffs not searched", + "old": "\"-m\",\n \"-1\",", + "new": "\"-1\",", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_score_added_only_in_a_merge_resolution_is_refused" + ] + }, + { + "mutation": "shallow history trusted", + "old": "if shallow.returncode != 0 or shallow.stdout.strip() != \"false\":", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed" + ] + }, + { + "mutation": "replacement refs honoured", + "old": "\"--no-replace-objects\",", + "new": "", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_replacement_ref_cannot_hide_a_committed_score" + ] + }, + { + "mutation": "SHA-256 ids rejected", + "old": "SHA_RX = re.compile(r\"[0-9a-f]{40}(?:[0-9a-f]{24})?\")", + "new": "SHA_RX = re.compile(r\"[0-9a-f]{40}\")", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" + ] + }, + { + "mutation": "subprocess OSError escapes", + "old": "except (ValueError, TypeError, OSError) as error:\n return subprocess.CompletedProcess(", + "new": "except ValueError as error:\n return subprocess.CompletedProcess(", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]", + "tests/test_track_b_holdout_registry.py::test_subprocess_failures_are_reported_not_raised" + ] + }, + { + "mutation": "authorization shapes unchecked", + "old": "problems.extend(_authorization_problems(imports))", + "new": "pass", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_authorizations_are_reported_not_raised[None-must be a mapping]", + "tests/test_track_b_holdout_registry.py::test_malformed_authorizations_are_reported_not_raised[imports1-needs a gate]", + "tests/test_track_b_holdout_registry.py::test_malformed_authorizations_are_reported_not_raised[imports2-needs]" + ] + }, + { + "mutation": "a new registry needs no admission", + "old": "before = previous if previous is not None else empty_like(current)", + "new": "before = previous if previous is not None else current", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_new_registry_is_admitted_against_an_empty_one" + ] + }, + { + "mutation": "history checks only the endpoint", + "old": "previous_commit, previous = commit, current", + "new": "previous_commit = commit", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_clean_history_has_no_problems", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]" + ] + }, + { + "mutation": "an unresolved push boundary falls back", + "old": " if _commit(repository, before) is None:\n raise LookupError(\n f\"the push's before commit {before} does not resolve\"\n )\n return before", + "new": " if _commit(repository, before) is not None:\n return before", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"}]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]" + ] + }, + { + "mutation": "a push naming no commit falls through", + "old": " raise LookupError(\"the push event names no before commit\")", + "new": " before = \"0\" * 40", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": \"HEAD\"}]", + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": null}]", + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{}]" + ] + }, + { + "mutation": "a non-object event is read", + "old": " if not isinstance(payload, Mapping):\n raise LookupError(\"the GitHub event is not a JSON object\")\n", + "new": "", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[[]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[[]" + ] + }, + { + "mutation": "an unreadable pull request event is skipped", + "old": " raise LookupError(\"the pull_request event names no head commit\")", + "new": " return None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{\"pull_request\": null}]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{\"pull_request\": {\"head\": \"abc\"}}]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{\"pull_request\": {\"head\": {\"sha\": \"HEAD\"}}}]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{}]" + ] + }, + { + "mutation": "binary git failures escape", + "old": " except (ValueError, TypeError, OSError):\n return None\n return found.stdout if found.returncode == 0 else None", + "new": " except ValueError:\n return None\n return found.stdout if found.returncode == 0 else None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure0]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure1]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure2]" + ] + } + ] +} diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index 83a0ca1f..e3f5e03f 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -324,6 +324,10 @@ # by B2's floor builder, comparator and scorer; nothing in the # historical projection imports it. Path("src/populace_dynamics/track_b/b2.py"), + # The held-out data registry's checks read only the registry, git and + # committed run artifacts; nothing in the historical projection + # imports them. + Path("src/populace_dynamics/track_b/holdout_registry.py"), # S1 development and future registration; no historical projection imports. Path("src/populace_dynamics/engine/forward_earnings_s1_v1.py"), # U2 documentary metadata stays outside the historical projection. diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py new file mode 100644 index 00000000..fd2f7e8b --- /dev/null +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -0,0 +1,1351 @@ +"""The held-out data registry and the rules every commit must keep. + +``docs/design/track_b_holdout_registry.json`` records the data that Track +B's gates and the open-population lane's gates hold out. The dynamics +coordinator made it the single registry for every lane on 2026-10-21. +This module enforces its rules: + +- one schema, ``track_b_holdout_registry.v1``, with every field typed; +- a source vocabulary held here, in :data:`SOURCES`, so that adding a + source or a family is a reviewed code change. Each source has a family, + a title, a catalog reference no other source shares, and what its key + year means. The registry's copy must equal it; +- unit keys ``source:years[:scope...]``, where ``years`` is a year, an + inclusive year range, or ``undated``. Two keys name a shared unit when + they have the same source and their years intersect (``undated`` + intersects every year). Scope segments describe the unit and never + narrow the overlap; +- live entries that share a unit, or whose relation is unresolved, list + each other in ``disclosed_overlaps``, and every disclosure is symmetric; +- relations the mechanical rule cannot see are reviewed: an entry lists + every earlier live entry of the same source family that it neither + shares a unit with nor discloses, with the outcome ``disjoint``, a + substantive reason and citations. An unresolved relation is disclosed + instead; +- unique entry ids, which are never removed, reordered or reused; +- histories with legal transitions only. A registration starts as a lone + ``candidate`` in a commit of its own, so it reaches the base branch + before its gate freezes, and its scored run's bytes must not yet have + been committed, at any path, when it is registered. A historical import + is a gate that scored before the registry existed: it starts as + ``used``, and its id, gate, unit keys and scored runs must match an + authorization in :data:`HISTORICAL_IMPORTS`; +- a ``used`` entry cites scored-run artifacts listed in + :data:`SCORED_RUNS`, which pins each one's gate and the SHA-256 of its + bytes, read from the committed tree; each citation's identity field + names the gate in ``gates.yaml`` or in + :data:`REGISTRATIONS_OUTSIDE_GATES_YAML`; +- entries only grow: against the base registry, between every pair of + consecutive first-parent commits since it (each snapshot validated + against its own commit's tree), and between a pull request's own + commits, an entry's identity and units never change and its lists only + extend. A newly created registry is admitted against an empty one; +- the registry stays incomplete until its completion record, B0.2's + mechanical exposure inventory, exists. + +The overlap rule is deliberately coarse. It catches two gates that read +the same source at the same year, whatever variable or table they use. +It cannot see that two sources share underlying data, or that two +editions of a publication report the same data years. That is what the +family review and disclosures are for, and what reviewers of each +registry change must check. + +Each check returns a list of problems; an empty list means the registry +keeps the rules. +""" + +from __future__ import annotations + +import datetime as dt +import hashlib +import json +import os +import re +import subprocess +from collections.abc import Mapping +from pathlib import Path +from types import MappingProxyType +from typing import Any + +import yaml + +SCHEMA = "track_b_holdout_registry.v1" +REGISTRY = "docs/design/track_b_holdout_registry.json" +GATES = "gates.yaml" + +#: The source vocabulary. Adding a source is a reviewed code change, and a +#: new id must not be another name for a declared source. +SOURCES: Mapping[str, Mapping[str, str]] = MappingProxyType( + { + "psid": { + "family": "psid", + "title": ( + "Panel Study of Income Dynamics: family files, cross-year " + "individual file and supplements" + ), + "catalog": ( + "University of Michigan Institute for Social Research: " + "Panel Study of Income Dynamics" + ), + "key_year": ( + "interview wave (collection year); income reference years " + "are the wave year minus one" + ), + }, + "ssa_epuf": { + "family": "ssa_administrative", + "title": "SSA Earnings Public-Use File", + "catalog": ( + "SSA Office of Research, Evaluation, and Statistics: " + "Earnings Public-Use File" + ), + "key_year": ( + "the vintage year in the file's title: the 2006 file holds " + "earnings through 2006 and was released in August 2011" + ), + }, + "ssa_supplement": { + "family": "ssa_administrative", + "title": ( + "Annual Statistical Supplement to the Social Security " + "Bulletin" + ), + "catalog": ( + "SSA Office of Research, Evaluation, and Statistics: Annual " + "Statistical Supplement to the Social Security Bulletin" + ), + "key_year": "edition year", + }, + "ssa_di_asr": { + "family": "ssa_administrative", + "title": ( + "Annual Statistical Report on the Social Security " + "Disability Insurance Program" + ), + "catalog": ( + "SSA Office of Research, Evaluation, and Statistics: Annual " + "Statistical Report on the Social Security Disability " + "Insurance Program" + ), + "key_year": "edition year", + }, + "census_cps_tables": { + "family": "us_cross_section", + "title": "U.S. Census Bureau Current Population Survey tables", + "catalog": ( + "U.S. Census Bureau: Current Population Survey published " + "tables" + ), + "key_year": "table reference year", + }, + "populace_us_frame": { + "family": "us_cross_section", + "title": "Microcosm US certified frame", + "catalog": "hf://policyengine/populace-us/populace_us_2024.h5", + "key_year": "frame vintage", + }, + "mermin_smith_anchors": { + "family": "literature", + "title": ( + "Mermin and Smith published orderings, committed as anchors" + ), + "catalog": ( + "gates.yaml gate_w1 holdout_basis: " + "mermin_smith_committed_orderings" + ), + "key_year": "undated", + }, + } +) +#: Gates registered outside ``gates.yaml`` (Track B's #42 registrations), +#: with every identifier that may name them in scored evidence. +REGISTRATIONS_OUTSIDE_GATES_YAML: Mapping[str, tuple[str, ...]] = ( + MappingProxyType({"B2": ("B2", "6252388925")}) +) +#: Each scored-run artifact a ``used`` entry may cite: its gate and the +#: SHA-256 of its committed bytes. A gate's scored run is added here when +#: it scores, as a reviewed code change, so a floor, candidate file or +#: dry run cannot stand in for a score, and a reused path cannot carry +#: another gate's run. +SCORED_RUNS: Mapping[str, tuple[str, str]] = MappingProxyType( + { + "runs/gate_m6_candidate1_v1.json": ( + "gate_m6", + "546a9739f8d1c7d21a91a07eb902c8af9bda92cdaa8f7917f312894f6a861b24", + ), + "runs/gate_m6_candidate2_v1.json": ( + "gate_m6", + "f217b0dc3b17b0a2c2926b1e9299ba2144be4b8a69ac880566d19fd4cd6c2917", + ), + "runs/gate_m6_candidate3_v1.json": ( + "gate_m6", + "caf254925f44b27c1bd1131336055e27fbc311daec00b0050b2f9293a74e82cf", + ), + "runs/track_b_b2_score.json": ( + "B2", + "df82a102a2b4c40c34f0e36b4333dbe7b90d13bfb3658b60b3f6d3f3e2dd8489", + ), + "runs/epuf_fill_gate_test_v1.json": ( + "gate_epuf_fill", + "9fa726be90134546c54148d681209ccf4a33939a7d6e2cfe5c2e384832dace03", + ), + "runs/gate_m4_hazard_v1.json": ( + "gate_m4", + "6493fda8207e743b3b6cbe1c80eb5ab5fb9646aa6f10fad4e77aa8d84c6b889c", + ), + "runs/gate_w1_candidate1_v1.json": ( + "gate_w1", + "01130c512ddba545e11dcf83364387a6c8df307041ac199ce3703cc4a00d6725", + ), + "runs/gate_w1_candidate2_v1.json": ( + "gate_w1", + "3baf097284b793fa4968d9b0b935ec3d16cf2770b16fb1fc1e3fb591ad490f40", + ), + "runs/gate_w1_candidate3_v1.json": ( + "gate_w1", + "b532f2b75d96e876af2e65400439a2f6c832ea45eecf83c1b4431b026ab7e688", + ), + "runs/gate_w1_candidate4_v1.json": ( + "gate_w1", + "ce4f9c9ab0b78bbb4dfe7dcb2a6b34892a80ea74760e0a00c4d60aa35e2d6e65", + ), + } +) +#: The historical imports this module authorizes: gates that scored before +#: the registry existed. Each binds one entry id to its gate, its unit keys +#: and its scored runs, so the exception to preregistration covers exactly +#: the exposure reviewed here. B0.2's mechanical inventory adds any import +#: it finds, or a correction to an existing one, as a new authorized entry +#: in a reviewed code change; nothing else can enter as an import. +HISTORICAL_IMPORTS: Mapping[str, Mapping[str, Any]] = MappingProxyType( + { + "H001": MappingProxyType( + { + "gate": "gate_m6", + "unit_keys": ("psid:2015-2023:held_out_window",), + "scored_runs": ( + "runs/gate_m6_candidate1_v1.json", + "runs/gate_m6_candidate2_v1.json", + "runs/gate_m6_candidate3_v1.json", + ), + } + ), + "H002": MappingProxyType( + { + "gate": "B2", + "unit_keys": ("psid:2013:fam2013er", "psid:2015:fam2015er"), + "scored_runs": ("runs/track_b_b2_score.json",), + } + ), + "H003": MappingProxyType( + { + "gate": "gate_epuf_fill", + "unit_keys": ("ssa_epuf:2006:test_partition",), + "scored_runs": ("runs/epuf_fill_gate_test_v1.json",), + } + ), + "H004": MappingProxyType( + { + "gate": "gate_m4", + "unit_keys": ( + "psid:1982:employment_status", + "psid:1983:employment_status", + "psid:1993-1997:employment_status", + "psid:1999-2023:employment_status", + "ssa_di_asr:2023", + ), + "scored_runs": ("runs/gate_m4_hazard_v1.json",), + } + ), + "H005": MappingProxyType( + { + "gate": "gate_w1", + "unit_keys": ( + "ssa_supplement:2023:6.b5.1", + "ssa_supplement:2023:6.b4", + "ssa_supplement:2023:6.b3", + "ssa_di_asr:2023:table_19", + "census_cps_tables:2023:hh-4", + "census_cps_tables:2023:ad-3", + "populace_us_frame:2024:certified", + "mermin_smith_anchors:undated", + ), + "scored_runs": ( + "runs/gate_w1_candidate1_v1.json", + "runs/gate_w1_candidate2_v1.json", + "runs/gate_w1_candidate3_v1.json", + "runs/gate_w1_candidate4_v1.json", + ), + } + ), + } +) +STATUSES = ("candidate", "frozen", "used", "withdrawn") +TRANSITIONS = { + "candidate": frozenset({"frozen", "withdrawn"}), + "frozen": frozenset({"used", "withdrawn"}), + "used": frozenset(), + "withdrawn": frozenset(), +} +#: Each provenance's permitted first statuses. +FIRST_STATUS = { + "registration": frozenset({"candidate"}), + "historical_import": frozenset({"used"}), +} +TEXT_FIELDS = ("schema", "title", "purpose", "completion_record") +TOP_LEVEL = ( + *TEXT_FIELDS, + "rules", + "status_values", + "sources", + "complete", + "entries", +) +SOURCE_FIELDS = ("family", "title", "catalog", "key_year") +ENTRY_TEXT = ("id", "provenance", "gate", "lane", "source", "status") +ENTRY_LISTS = ( + "gate_identifiers", + "unit_keys", + "evidence", + "scored_evidence", + "disclosed_overlaps", + "overlap_review", + "history", +) +ENTRY_KEYS = (*ENTRY_TEXT, "units", *ENTRY_LISTS) +#: Fields that identify an entry and its units; they never change. +IMMUTABLE = ( + "id", + "provenance", + "gate", + "lane", + "source", + "units", + "unit_keys", +) +#: Lists that may only extend. +EXTENDING = ( + "gate_identifiers", + "evidence", + "scored_evidence", + "disclosed_overlaps", + "overlap_review", + "history", +) +ITEM_FIELDS = { + "scored_evidence": ("path", "field", "equals"), + "overlap_review": ("id", "outcome", "reason"), + "history": ("date", "status", "note"), +} +#: Identity fields a scored-run artifact may be matched on. +IDENTITY_FIELDS = ("gate", "gate.name", "registration_id") +REVIEW_OUTCOMES = ("disjoint",) +MIN_REASON = 40 +PLACEHOLDER_RX = re.compile( + r"\b(todo|tbd|fixme|xxx|placeholder|to be determined)\b", re.I +) +ID_RX = re.compile(r"H[0-9]{3,}") +SOURCE_RX = re.compile(r"[a-z][a-z0-9_]*") +YEARS_RX = re.compile(r"(?P[0-9]{4})(?:-(?P[0-9]{4}))?|undated") +SCOPE_RX = re.compile(r"[a-z0-9][a-z0-9_.\-]*") +#: Scored-run artifacts live here. +RUNS_DIR = "runs" +RUN_PATH_RX = re.compile(rf"{RUNS_DIR}/[A-Za-z0-9_.\-]+\.json") +#: A repository-relative path: no leading slash, no NUL, no ``..``. +PATH_RX = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_./\-]*") +CITATION_RX = re.compile( + r"(?P[A-Za-z0-9_][A-Za-z0-9_./\-]*):(?P[0-9]{1,7})" + r"(?:-(?P[0-9]{1,7}))?" +) +URL_RX = re.compile(r"https://\S+") +#: A full object id: SHA-1 (40 hex digits) or SHA-256 (64). +SHA_RX = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?") + + +def load(repository: Path) -> dict[str, Any]: + """The registry in ``repository``'s working tree.""" + return json.loads((repository / REGISTRY).read_text(encoding="utf-8")) + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def parse_key(key: Any) -> tuple[str, tuple[int, int] | None] | None: + """``(source, years)`` for a well-formed unit key, with ``years`` an + inclusive range or ``None`` for ``undated``; ``None`` otherwise.""" + if not isinstance(key, str): + return None + parts = key.split(":") + if len(parts) < 2 or not SOURCE_RX.fullmatch(parts[0]): + return None + if not all(SCOPE_RX.fullmatch(part) for part in parts[2:]): + return None + years = YEARS_RX.fullmatch(parts[1]) + if years is None: + return None + if parts[1] == "undated": + return parts[0], None + start = int(years["start"]) + end = int(years["end"] or start) + if end < start: + return None + return parts[0], (start, end) + + +def keys_overlap(a: str, b: str) -> bool: + """Whether two well-formed unit keys name a shared unit: the same + source, with intersecting years.""" + left, right = parse_key(a), parse_key(b) + if left is None or right is None or left[0] != right[0]: + return False + if left[1] is None or right[1] is None: + return True + return left[1][0] <= right[1][1] and right[1][0] <= left[1][1] + + +def entries_overlap(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: + """Whether two entries hold out a shared unit.""" + return any( + keys_overlap(x, y) for x in a["unit_keys"] for y in b["unit_keys"] + ) + + +def families(entry: Mapping[str, Any], sources: Mapping[str, Any]) -> set[str]: + """The source families an entry's unit keys belong to.""" + return { + sources[parsed[0]]["family"] + for parsed in map(parse_key, entry["unit_keys"]) + if parsed is not None and parsed[0] in sources + } + + +_MISSING = object() + + +def lookup(document: Any, dotted: str) -> Any: + """``document``'s value at a dotted field path, or a sentinel.""" + for part in dotted.split("."): + if not isinstance(document, Mapping) or part not in document: + return _MISSING + document = document[part] + return document + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: + """Run git. An argument git cannot take (a NUL byte, ``None``, an + argument list too long) or a missing git fails like a git error + rather than raising.""" + command = ["git", "-C", str(repository), *args] + try: + return subprocess.run( + command, capture_output=True, text=True, check=False + ) + except (ValueError, TypeError, OSError) as error: + return subprocess.CompletedProcess(command, 128, "", str(error)) + + +def _git_bytes( + repository: Path, *args: str, stdin: bytes | None = None +) -> bytes | None: + try: + found = subprocess.run( + ["git", "-C", str(repository), *args], + input=stdin, + capture_output=True, + check=False, + ) + except (ValueError, TypeError, OSError): + return None + return found.stdout if found.returncode == 0 else None + + +def safe_path(path: Any) -> bool: + """Whether ``path`` is a plain repository-relative path.""" + return ( + isinstance(path, str) + and PATH_RX.fullmatch(path) is not None + and ".." not in path.split("/") + ) + + +def _parse_registry(text: str | None) -> Any: + """A registry document, or ``None`` when ``text`` is not JSON.""" + if text is None: + return None + try: + return json.loads(text) + except (ValueError, RecursionError): + return None + + +class Committed: + """Read-only access to one commit of ``repository``, cached.""" + + def __init__(self, repository: Path, commit: str = "HEAD") -> None: + self.repository = repository + self.commit = commit + self._files: dict[str, bytes | None] = {} + self._gates: Mapping[str, Any] | None = None + self._registered: dict[str, str | None] | None = None + + def blob(self, path: str, commit: str | None = None) -> bytes | None: + """A regular committed file's bytes at ``commit`` (by default this + one's), or ``None`` for a symlink, a directory, a missing path or + an unsafe one.""" + commit = commit or self.commit + key = f"{commit}:{path}" + if key not in self._files: + content = None + if safe_path(path): + listed = _git(self.repository, "ls-tree", commit, "--", path) + fields = listed.stdout.split() + if ( + listed.returncode == 0 + and len(fields) >= 4 + and fields[0] in ("100644", "100755") + and fields[1] == "blob" + ): + content = _git_bytes(self.repository, "show", key) + self._files[key] = content + return self._files[key] + + def text(self, path: str) -> str | None: + """A regular committed UTF-8 file's text, or ``None``.""" + content = self.blob(path) + if content is None: + return None + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return None + + def identifiers(self, gate: str) -> set[str]: + """Every identifier that may name ``gate`` in scored evidence.""" + if self._gates is None: + text = self.text(GATES) + try: + document = yaml.safe_load(text) if text else {} + except (yaml.YAMLError, ValueError, RecursionError): + document = {} + gates = ( + document.get("gates") if isinstance(document, dict) else None + ) + self._gates = gates if isinstance(gates, Mapping) else {} + names = set(REGISTRATIONS_OUTSIDE_GATES_YAML.get(gate, ())) + declared = self._gates.get(gate) + if isinstance(declared, Mapping): + names.add(gate) + for key in ("id", "registration_id"): + if isinstance(declared.get(key), str): + names.add(declared[key]) + return names + + def registered_at(self, entry_id: str) -> str | None: + """The oldest commit on this commit's first-parent chain whose + registry holds ``entry_id``, or ``None``.""" + if self._registered is None: + self._registered = {} + log = _git( + self.repository, + "log", + "--first-parent", + "--reverse", + "--format=%H", + self.commit, + "--", + REGISTRY, + ) + for commit in log.stdout.split() if log.returncode == 0 else []: + document = _parse_registry( + _decode( + _git_bytes( + self.repository, "show", f"{commit}:{REGISTRY}" + ) + ) + ) + entries = ( + document.get("entries") + if isinstance(document, Mapping) + else None + ) + for entry in entries if isinstance(entries, list) else []: + name = ( + entry.get("id") if isinstance(entry, Mapping) else None + ) + if isinstance(name, str): + self._registered.setdefault(name, commit) + return self._registered.get(entry_id) + + def committed_by(self, content: bytes, commit: str) -> bool | None: + """Whether a file with exactly ``content`` was committed, at any + path and whether or not it was later moved or deleted, in + ``commit`` or any of its ancestors. ``None`` when git cannot say, + including when a shallow clone hides ancestors: an empty search + proves nothing there. Replacement refs are ignored, so the search + reads the recorded history.""" + name = _git_bytes( + self.repository, "hash-object", "--stdin", stdin=content + ) + if not name or not name.strip(): + return None + found = _git( + self.repository, + "--no-replace-objects", + "log", + "-m", + "-1", + "--format=%H", + f"--find-object={name.decode().strip()}", + commit, + ) + if found.returncode != 0: + return None + if found.stdout.strip(): + return True + shallow = _git(self.repository, "rev-parse", "--is-shallow-repository") + if shallow.returncode != 0 or shallow.stdout.strip() != "false": + return None + return False + + +def _decode(content: bytes | None) -> str | None: + if content is None: + return None + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return None + + +def _authorization_problems(imports: Any) -> list[str]: + """Problems with the shape of the historical-import authorizations.""" + if not isinstance(imports, Mapping): + return ["imports: the authorizations must be a mapping"] + problems = [] + for name, authorized in imports.items(): + if not ( + isinstance(authorized, Mapping) + and isinstance(authorized.get("gate"), str) + and all( + isinstance(authorized.get(key), (list, tuple)) + and all(isinstance(item, str) for item in authorized[key]) + for key in ("unit_keys", "scored_runs") + ) + ): + problems.append( + f"imports: {name!r} needs a gate, unit_keys and scored_runs" + ) + return problems + + +def _import_problems( + entry: Mapping[str, Any], imports: Mapping[str, Mapping[str, Any]] +) -> list[str]: + """Problems with a historical import against its authorization.""" + name = entry["id"] + authorized = imports.get(name) + if authorized is None: + return [f"{name}: is not an authorized historical import"] + problems = [] + if entry["gate"] != authorized["gate"]: + problems.append( + f"{name}: is authorized for {authorized['gate']!r}, not " + f"{entry['gate']!r}" + ) + if sorted(entry["unit_keys"]) != sorted(authorized["unit_keys"]): + problems.append(f"{name}: unit keys differ from its authorization") + cited = sorted(item["path"] for item in entry["scored_evidence"]) + if cited != sorted(authorized["scored_runs"]): + problems.append( + f"{name}: scored evidence differs from its authorized runs" + ) + return problems + + +def _evidence_problems( + entry: Mapping[str, Any], + committed: Committed | None, + scored_runs: Mapping[str, tuple[str, str]], + imports: Mapping[str, Mapping[str, Any]], +) -> list[str]: + """Problems with an entry's scored-evidence citations and, for a + historical import, with its authorization.""" + name = entry["id"] + problems: list[str] = [] + if entry["provenance"] == "historical_import": + problems.extend(_import_problems(entry, imports)) + if not entry["scored_evidence"]: + return problems + allowed = set(entry["gate_identifiers"]) + if committed is not None: + authoritative = committed.identifiers(entry["gate"]) + if not authoritative: + return [ + *problems, + f"{name}: {entry['gate']!r} is neither a gates.yaml gate nor " + "a registration this module lists", + ] + allowed &= authoritative + registered = None + if committed is not None and entry["provenance"] == "registration": + registered = committed.registered_at(name) + if registered is None: + problems.append( + f"{name}: a registration cites scored evidence before it is " + "registered on the base branch" + ) + for item in entry["scored_evidence"]: + path, field, equals = item["path"], item["field"], item["equals"] + run = scored_runs.get(path) + if run is None: + problems.append( + f"{name}: scored evidence {path!r} is not a listed scored run" + ) + continue + if run[0] != entry["gate"]: + problems.append( + f"{name}: scored evidence {path!r} is {run[0]!r}'s scored run" + ) + continue + if field not in IDENTITY_FIELDS: + problems.append(f"{name}: {field!r} is not an identity field") + continue + if equals not in allowed: + problems.append( + f"{name}: scored evidence names {equals!r}, which does not " + "identify this gate" + ) + continue + if committed is None: + continue + content = committed.blob(path) + if content is None: + problems.append( + f"{name}: scored evidence {path!r} is not a committed file" + ) + continue + if hashlib.sha256(content).hexdigest() != run[1]: + problems.append( + f"{name}: scored evidence {path!r} differs from the listed " + "scored run's bytes" + ) + continue + document = _parse_registry(_decode(content)) + if lookup(document, field) != equals: + problems.append( + f"{name}: scored evidence {path!r} does not have " + f"{field} == {equals!r}" + ) + continue + if registered is not None: + earlier = committed.committed_by(content, registered) + if earlier is None: + problems.append( + f"{name}: cannot tell whether its scored run {path!r} " + "was committed before it was registered" + ) + elif earlier: + problems.append( + f"{name}: its scored run {path!r} was already committed, " + "at some path, when the entry was registered" + ) + return problems + + +def _review_problems( + entry: Mapping[str, Any], committed: Committed | None +) -> list[str]: + """Problems with an entry's family-review items.""" + name = entry["id"] + problems: list[str] = [] + for item in entry["overlap_review"]: + reviewed = item["id"] + if item["outcome"] not in REVIEW_OUTCOMES: + problems.append( + f"{name}: review of {reviewed} has outcome " + f"{item['outcome']!r}; an unresolved relation is disclosed" + ) + reason = item["reason"].strip() + if len(reason) < MIN_REASON or PLACEHOLDER_RX.search(reason): + problems.append( + f"{name}: review of {reviewed} needs a substantive reason" + ) + citations = item.get("evidence") + if not isinstance(citations, list) or not citations: + problems.append(f"{name}: review of {reviewed} cites nothing") + continue + for citation in citations: + problems.extend(_citation_problems(name, citation, committed)) + return problems + + +def _citation_problems( + name: str, citation: Any, committed: Committed | None +) -> list[str]: + if not isinstance(citation, str): + return [f"{name}: a review citation is not a string"] + if URL_RX.fullmatch(citation): + return [] + match = CITATION_RX.fullmatch(citation) + if match is None: + return [f"{name}: citation {citation!r} is not path:line or a URL"] + start = int(match["start"]) + end = int(match["end"] or start) + if not 1 <= start <= end: + return [f"{name}: citation {citation!r} has a bad line range"] + if committed is None: + return [] + text = committed.text(match["path"]) + if text is None: + return [f"{name}: citation {citation!r} names no committed file"] + if end > len(text.splitlines()): + return [f"{name}: citation {citation!r} runs past the file's end"] + return [] + + +def _shape_problems(entry: Any) -> list[str]: + """Type problems in one entry; later checks assume none.""" + if not isinstance(entry, Mapping): + return ["registry: an entry is not a mapping"] + name = entry.get("id") if _text(entry.get("id")) else "" + missing = [key for key in ENTRY_KEYS if key not in entry] + if missing: + return [f"{name}: missing {', '.join(missing)}"] + problems = [ + f"{name}: {key} must be a non-empty string" + for key in ENTRY_TEXT + if not _text(entry[key]) + ] + problems += [ + f"{name}: {key} must be a list" + for key in ENTRY_LISTS + if not isinstance(entry[key], list) + ] + if not isinstance(entry["units"], Mapping) or not entry["units"]: + problems.append(f"{name}: units must be a non-empty mapping") + if problems: + return problems + plain = ("gate_identifiers", "unit_keys", "evidence", "disclosed_overlaps") + for key in plain: + if not all(_text(item) for item in entry[key]): + problems.append(f"{name}: {key} must hold non-empty strings") + for key, fields in ITEM_FIELDS.items(): + for item in entry[key]: + if not isinstance(item, Mapping) or not all( + _text(item.get(field)) for field in fields + ): + problems.append( + f"{name}: each {key} item needs non-empty " + f"{', '.join(fields)}" + ) + break + return problems + + +def _entry_problems( + entry: Mapping[str, Any], + sources: Mapping[str, Any], + committed: Committed | None, + scored_runs: Mapping[str, tuple[str, str]], + imports: Mapping[str, Mapping[str, Any]], +) -> list[str]: + """Problems inside one well-shaped entry.""" + name = entry["id"] + problems: list[str] = [] + if not ID_RX.fullmatch(name): + problems.append(f"{name}: id must be H followed by 3+ digits") + if entry["provenance"] not in FIRST_STATUS: + problems.append(f"{name}: unknown provenance") + return problems + for key in ("gate_identifiers", "unit_keys", "evidence"): + if not entry[key]: + problems.append(f"{name}: {key} must not be empty") + elif len(set(entry[key])) != len(entry[key]): + problems.append(f"{name}: {key} repeat") + for key in entry["unit_keys"]: + parsed = parse_key(key) + if parsed is None: + problems.append(f"{name}: malformed unit key {key!r}") + elif parsed[0] not in sources: + problems.append(f"{name}: unit key {key!r} names no source") + history = entry["history"] + if not history: + problems.append(f"{name}: history must not be empty") + return problems + first = FIRST_STATUS[entry["provenance"]] + if history[0]["status"] not in first: + problems.append( + f"{name}: a {entry['provenance']} starts as one of " + f"{sorted(first)}, not {history[0]['status']!r}" + ) + previous_status, previous_date = None, None + for index, item in enumerate(history): + status = item["status"] + if status not in STATUSES: + problems.append(f"{name}: history[{index}] status {status!r}") + return problems + try: + date = dt.date.fromisoformat(item["date"]) + except ValueError: + problems.append(f"{name}: history[{index}] date is not a date") + date = previous_date + if previous_date is not None and date < previous_date: + problems.append(f"{name}: history[{index}] goes back in time") + previous_date = date + if previous_status is not None and ( + status not in TRANSITIONS[previous_status] + ): + problems.append( + f"{name}: history[{index}] moves {previous_status} -> " + f"{status}, which is not a legal transition" + ) + previous_status = status + if entry["status"] != previous_status: + problems.append( + f"{name}: status {entry['status']!r} is not its history's " + f"last status {previous_status!r}" + ) + if entry["status"] == "used" and not entry["scored_evidence"]: + problems.append(f"{name}: a used entry needs scored_evidence") + problems.extend(_evidence_problems(entry, committed, scored_runs, imports)) + problems.extend(_review_problems(entry, committed)) + return problems + + +def _vocabulary_problems(sources: Any) -> list[str]: + if not isinstance(sources, Mapping) or not sources: + return ["registry: sources must be a non-empty mapping"] + problems: list[str] = [] + catalogs: dict[str, str] = {} + for source, declared in sources.items(): + if not isinstance(source, str) or not SOURCE_RX.fullmatch(source): + problems.append(f"source {source!r}: malformed id") + if not isinstance(declared, Mapping) or not all( + _text(declared.get(field)) for field in SOURCE_FIELDS + ): + problems.append( + f"source {source!r}: needs {', '.join(SOURCE_FIELDS)}" + ) + continue + catalog = " ".join(declared["catalog"].split()).casefold() + if catalog in catalogs: + problems.append( + f"source {source!r}: shares its catalog with " + f"{catalogs[catalog]!r}, so one of them is an alias" + ) + catalogs[catalog] = source + return problems + + +def validate( + registry: Any, + repository: Path | None = None, + vocabulary: Mapping[str, Mapping[str, str]] = SOURCES, + *, + commit: str = "HEAD", + scored_runs: Mapping[str, tuple[str, str]] = SCORED_RUNS, + imports: Mapping[str, Mapping[str, Any]] = HISTORICAL_IMPORTS, +) -> list[str]: + """Every rule one registry must keep on its own. ``vocabulary`` is the + controlled source list the registry's copy must equal; + ``scored_runs`` and ``imports`` are the controlled lists of scored + runs and authorized historical imports. With ``repository``, scored + evidence, citations and the completion record are checked against its + ``commit``.""" + if not isinstance(registry, Mapping): + return ["registry: not a mapping"] + missing = [key for key in TOP_LEVEL if key not in registry] + if missing: + return [f"registry: missing {', '.join(missing)}"] + problems = [ + f"registry: {key} must be a non-empty string" + for key in TEXT_FIELDS + if not _text(registry[key]) + ] + if problems: + return problems + if registry["schema"] != SCHEMA: + problems.append(f"registry: schema {registry['schema']!r}") + rules = registry["rules"] + if not isinstance(rules, list) or not rules: + problems.append("registry: rules must be a non-empty list") + elif not all(_text(rule) for rule in rules): + problems.append("registry: each rule must be a non-empty string") + values = registry["status_values"] + if ( + not isinstance(values, Mapping) + or set(values) != set(STATUSES) + or not all(_text(text) for text in values.values()) + ): + problems.append("registry: status_values must describe each status") + if not isinstance(registry["complete"], bool): + problems.append("registry: complete must be true or false") + problems.extend(_vocabulary_problems(vocabulary)) + problems.extend(_authorization_problems(imports)) + if not isinstance(vocabulary, Mapping) or registry["sources"] != dict( + vocabulary + ): + problems.append( + "registry: sources must equal the controlled vocabulary" + ) + entries = registry["entries"] + if not isinstance(entries, list): + problems.append("registry: entries must be a list") + if problems: + return problems + for entry in entries: + problems.extend(_shape_problems(entry)) + if problems: + return problems + committed = ( + Committed(repository, commit) if repository is not None else None + ) + record = registry["completion_record"] + if not safe_path(record): + return [f"registry: completion_record {record!r} is not a path"] + if ( + committed is not None + and registry["complete"] + and committed.blob(record) is None + ): + problems.append( + f"registry: marked complete, but {record} is not committed" + ) + for entry in entries: + problems.extend( + _entry_problems(entry, vocabulary, committed, scored_runs, imports) + ) + if problems: + return problems + ids = [entry["id"] for entry in entries] + if len(set(ids)) != len(ids): + return ["registry: entry ids repeat"] + owners: dict[str, str] = {} + for entry in entries: + for identifier in entry["gate_identifiers"]: + owner = owners.setdefault(identifier, entry["gate"]) + if owner != entry["gate"]: + problems.append( + f"{entry['id']}: gate identifier {identifier!r} " + f"already identifies {owner!r}" + ) + by_id = {entry["id"]: entry for entry in entries} + for entry in entries: + for other in entry["disclosed_overlaps"]: + if other == entry["id"]: + problems.append(f"{entry['id']}: discloses itself") + elif other not in by_id: + problems.append(f"{entry['id']}: discloses unknown {other}") + elif entry["id"] not in by_id[other]["disclosed_overlaps"]: + problems.append( + f"{entry['id']}: discloses {other}, which does not " + "disclose it back" + ) + live = [entry for entry in entries if entry["status"] != "withdrawn"] + for index, entry in enumerate(live): + for other in live[index + 1 :]: + if entries_overlap(entry, other) and ( + other["id"] not in entry["disclosed_overlaps"] + ): + problems.append( + f"{entry['id']} and {other['id']} hold out a shared " + "unit without disclosing it" + ) + position = {name: index for index, name in enumerate(ids)} + for index, entry in enumerate(entries): + reviewed = {item["id"] for item in entry["overlap_review"]} + for name in reviewed: + if position.get(name, index) >= index: + problems.append( + f"{entry['id']}: reviews {name}, which is not an " + "earlier entry" + ) + if entry["status"] == "withdrawn": + continue + mine = families(entry, vocabulary) + for other in entries[:index]: + if ( + other["status"] != "withdrawn" + and mine & families(other, vocabulary) + and not entries_overlap(entry, other) + and other["id"] not in entry["disclosed_overlaps"] + and other["id"] not in reviewed + ): + problems.append( + f"{entry['id']}: shares a source family with " + f"{other['id']} but neither discloses nor reviews it" + ) + return problems + + +def _extends(base: Any, current: Any) -> bool: + """``current`` is a list that keeps the list ``base`` as its prefix.""" + return ( + isinstance(base, list) + and isinstance(current, list) + and current[: len(base)] == base + ) + + +def empty_like(registry: Mapping[str, Any]) -> dict[str, Any]: + """The registry as it stood before its first entry: the base a newly + created registry is admitted against.""" + return { + "schema": registry.get("schema"), + "status_values": registry.get("status_values"), + "completion_record": registry.get("completion_record"), + "rules": [], + "sources": {}, + "complete": False, + "entries": [], + } + + +def append_only_problems( + base: Mapping[str, Any], current: Mapping[str, Any] +) -> list[str]: + """How ``current`` breaks the append-only rule against ``base``, an + earlier registry that kept the rules.""" + if not isinstance(base, Mapping) or not isinstance(current, Mapping): + return ["registry: an append-only comparison needs two mappings"] + problems: list[str] = [] + for key in ("schema", "status_values", "completion_record"): + if current.get(key) != base.get(key): + problems.append(f"registry: {key} changed") + if not _extends(base.get("rules", []), current.get("rules", [])): + problems.append("registry: rules may only be appended") + sources = current.get("sources") + if not isinstance(sources, Mapping): + sources = {} + base_sources = base.get("sources") + if not isinstance(base_sources, Mapping): + base_sources = {} + for source, declared in base_sources.items(): + if sources.get(source) != declared: + problems.append(f"source {source!r}: changed or removed") + if base.get("complete") is True and current.get("complete") is not True: + problems.append("registry: a complete registry became incomplete") + old = base.get("entries") + new = current.get("entries") + for entries in (old, new): + if not isinstance(entries, list) or not all( + isinstance(entry, Mapping) and isinstance(entry.get("id"), str) + for entry in entries + ): + return [*problems, "registry: entries are malformed"] + old_ids = [entry.get("id") for entry in old] + new_ids = [entry.get("id") for entry in new] + if new_ids[: len(old_ids)] != old_ids: + problems.append( + "registry: earlier entries were removed, reordered or replaced" + ) + return problems + for before, after in zip(old, new[: len(old)], strict=True): + for key in IMMUTABLE: + if after.get(key) != before.get(key): + problems.append(f"{before['id']}: {key} changed") + for key in EXTENDING: + if not _extends(before.get(key, []), after.get(key, [])): + problems.append(f"{before['id']}: {key} may only extend") + for entry in new[len(old) :]: + name = entry.get("id") + history = entry.get("history") + statuses = ( + [item.get("status") for item in history] + if isinstance(history, list) + and all(isinstance(item, Mapping) for item in history) + else None + ) + if entry.get("provenance") == "historical_import": + if base.get("complete") is True: + problems.append( + f"{name}: historical imports close when the registry " + "is complete" + ) + elif statuses != ["candidate"]: + problems.append( + f"{name}: a new registration is added as a lone candidate, " + "and moves on only in a later commit" + ) + return problems + + +def _commit(repository: Path, ref: str) -> str | None: + found = _git( + repository, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}" + ) + return found.stdout.strip() if found.returncode == 0 else None + + +def _event() -> Mapping[str, Any]: + """The GitHub event payload at ``GITHUB_EVENT_PATH``. Raises + ``LookupError`` when it cannot be read or is not a JSON object.""" + path = os.environ.get("GITHUB_EVENT_PATH") + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except (TypeError, ValueError, OSError) as error: + raise LookupError(f"cannot read the GitHub event: {error}") from None + if not isinstance(payload, Mapping): + raise LookupError("the GitHub event is not a JSON object") + return payload + + +def base_commit(repository: Path) -> str: + """The commit the registry is compared against. + + - On a push (``GITHUB_EVENT_NAME=push``), the event's ``before`` + commit, which was the branch tip before the push. An all-zero + ``before`` (a branch's first push) falls through to the merge base. + - Otherwise, the merge base of ``HEAD`` and the base branch + (``GITHUB_BASE_REF`` on a pull request, otherwise ``master``), + preferring its ``origin/`` ref. + - When that merge base is ``HEAD`` itself, as on the base branch, its + first parent. + + Raises ``LookupError`` when none of these resolves, and on a push + whose event cannot be read or whose ``before`` commit does not + resolve: a push is compared with its own boundary, never with a + nearer commit that would check less history.""" + head = _commit(repository, "HEAD") + if head is None: + raise LookupError(f"no HEAD in {repository}") + if os.environ.get("GITHUB_EVENT_NAME") == "push": + before = _event().get("before") + if not isinstance(before, str) or SHA_RX.fullmatch(before) is None: + raise LookupError("the push event names no before commit") + if set(before) != {"0"}: + if _commit(repository, before) is None: + raise LookupError( + f"the push's before commit {before} does not resolve" + ) + return before + branch = os.environ.get("GITHUB_BASE_REF") or "master" + for ref in (f"origin/{branch}", branch): + found = _git(repository, "merge-base", "HEAD", ref) + if found.returncode == 0: + base = found.stdout.strip() + break + else: + raise LookupError(f"no merge base with {branch} in {repository}") + if base == head: + parent = _commit(repository, "HEAD^1") + if parent is None: + raise LookupError("HEAD is its own base and has no parent") + return parent + return base + + +def registry_at(repository: Path, commit: str) -> Any: + """The registry document at ``commit``, or ``None`` where it does not + exist. Raises ``LookupError`` when ``commit`` cannot be read and + ``ValueError`` when the registry there is not JSON.""" + listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) + if listed.returncode != 0: + raise LookupError(f"cannot list {REGISTRY} at {str(commit)[:80]}") + if not listed.stdout.strip(): + return None + document = _parse_registry( + _decode(_git_bytes(repository, "show", f"{commit}:{REGISTRY}")) + ) + if document is None: + raise ValueError(f"{REGISTRY} at {commit} is not JSON") + return document + + +def base_registry(repository: Path) -> Any: + """The registry at :func:`base_commit`, or ``None`` when the base has + no registry yet. Raises ``LookupError`` when no base resolves.""" + return registry_at(repository, base_commit(repository)) + + +def pull_request_head() -> str | None: + """The pull request's head commit, on a ``pull_request`` event, and + ``None`` on any other. Raises ``LookupError`` on a ``pull_request`` + event that cannot be read or names no head commit, so the pull + request's own commits are never silently left unchecked.""" + if os.environ.get("GITHUB_EVENT_NAME") != "pull_request": + return None + pull_request = _event().get("pull_request") + head = ( + pull_request.get("head") if isinstance(pull_request, Mapping) else None + ) + sha = head.get("sha") if isinstance(head, Mapping) else None + if not isinstance(sha, str) or SHA_RX.fullmatch(sha) is None: + raise LookupError("the pull_request event names no head commit") + return sha + + +def history_problems( + repository: Path, + base: str, + head: str = "HEAD", + *, + full: bool = True, + scored_runs: Mapping[str, tuple[str, str]] = SCORED_RUNS, + imports: Mapping[str, Mapping[str, Any]] = HISTORICAL_IMPORTS, +) -> list[str]: + """Problems across every first-parent commit from ``base`` to + ``head``, so a run of commits cannot hide a change a later commit + undoes. + + With ``full``, each snapshot is validated against its own commit's + tree. A snapshot that fails is reported and is not used as the next + one's predecessor. Each valid snapshot must append only to the last + valid one; a newly created registry is admitted against an empty + registry. Without ``full``, as for a pull request's own commits, only + the append-only rule is checked.""" + listed = _git( + repository, + "rev-list", + "--first-parent", + "--reverse", + f"{base}..{head}", + ) + if listed.returncode != 0: + raise LookupError(f"cannot list commits from {base} to {head}") + problems: list[str] = [] + try: + previous = registry_at(repository, base) + except ValueError as error: + problems.append(f"{base[:12]}: {error}") + previous = None + if not isinstance(previous, Mapping): + previous = None + previous_commit = base + for commit in listed.stdout.split(): + label = commit[:12] + try: + current = registry_at(repository, commit) + except ValueError as error: + problems.append(f"{label}: {error}") + continue + if current is None: + if previous is not None: + problems.append(f"{label}: the registry was deleted") + continue + if not isinstance(current, Mapping): + problems.append(f"{label}: registry: not a mapping") + continue + if full: + sources = current.get("sources") + shape = validate( + current, + repository, + vocabulary=sources if isinstance(sources, Mapping) else {}, + commit=commit, + scored_runs=scored_runs, + imports=imports, + ) + if shape: + problems.extend(f"{label}: {p}" for p in shape) + continue + before = previous if previous is not None else empty_like(current) + changes = append_only_problems(before, current) + if changes: + problems.extend( + f"{previous_commit[:12]}..{label}: {p}" for p in changes + ) + continue + previous_commit, previous = commit, current + return problems diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 1f5976c1..52f601fc 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,9 +39,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 18,125 | -| `artifact` | 4,261 | +| `unit` | 18,254 | +| `artifact` | 4,271 | | `integration_psid` | 1,450 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 308 | -| **Total** | **24,664** | +| **Total** | **24,803** | diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index 3a0a2086..42c3a50e 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -188,6 +188,7 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/track_b/arf.py"), Path("src/populace_dynamics/track_b/recompute.py"), Path("src/populace_dynamics/track_b/b2.py"), + Path("src/populace_dynamics/track_b/holdout_registry.py"), Path("src/populace_dynamics/engine/forward_earnings_s1_v1.py"), Path("src/populace_dynamics/data/u2_source_registry.py"), Path("scripts/capture_track_u2_income_wealth.py"), @@ -393,6 +394,7 @@ def test_post_review_exclusions_are_unreachable_from_birth_evidence(): "populace_dynamics.track_b.arf", "populace_dynamics.track_b.recompute", "populace_dynamics.track_b.b2", + "populace_dynamics.track_b.holdout_registry", "populace_dynamics.engine.forward_earnings_s1_v1", } assert track_b_modules.issubset(module_paths) diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py new file mode 100644 index 00000000..2deea18c --- /dev/null +++ b/tests/test_track_b_holdout_registry.py @@ -0,0 +1,1551 @@ +"""The held-out data registry's rules, on invented registries. + +``populace_dynamics.track_b.holdout_registry`` states the rules of the +single held-out data registry (dynamics coordinator, 2026-10-21). These +tests exercise them on invented registries and invented git repositories, +with the expected families and overlaps worked out here, independently +of the module. The committed registry itself is checked in +``test_track_b_holdout_registry_committed.py``. +""" + +from __future__ import annotations + +import copy +import errno +import hashlib +import json +import os +import subprocess +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from populace_dynamics.track_b import holdout_registry as registry + +#: Specified here, independently of the module, so that dropping a field +#: from the module's lists fails a test. +EXPECTED_IMMUTABLE = ( + "id", + "provenance", + "gate", + "lane", + "source", + "units", + "unit_keys", +) +EXPECTED_EXTENDING = ( + "gate_identifiers", + "evidence", + "scored_evidence", + "disclosed_overlaps", + "overlap_review", + "history", +) +VOCABULARY = { + "s_a": {"family": "f1", "title": "A", "catalog": "cat a", "key_year": "y"}, + "s_b": {"family": "f1", "title": "B", "catalog": "cat b", "key_year": "y"}, + "s_c": {"family": "f2", "title": "C", "catalog": "cat c", "key_year": "y"}, +} +#: The family of each invented source, independently of the module. +FAMILY = {"s_a": "f1", "s_b": "f1", "s_c": "f2"} +KEY_POOL = ( + "s_a:2020", + "s_a:2020-2022:x", + "s_a:2023", + "s_a:undated", + "s_b:2021:t1", + "s_b:2025", + "s_c:2020", + "s_c:2030-2031", +) +PATHS = { + ("registration", "candidate"): ["candidate"], + ("registration", "frozen"): ["candidate", "frozen"], + ("registration", "used"): ["candidate", "frozen", "used"], + ("registration", "withdrawn"): ["candidate", "withdrawn"], + ("historical_import", "used"): ["used"], +} +REASON = ( + "The two units come from different years of separate sources, so no " + "observation is shared." +) + + +def run_path(name: str) -> str: + return f"{registry.RUNS_DIR}/{name}.json" + + +def key_years(key: str): + """The source and the inclusive year range of an invented key.""" + source, years = key.split(":")[:2] + if years == "undated": + return source, None + start, _, end = years.partition("-") + return source, (int(start), int(end or start)) + + +def share_unit(a: dict, b: dict) -> bool: + """Whether two entries share a unit, worked out independently.""" + for x in a["unit_keys"]: + for y in b["unit_keys"]: + (sx, yx), (sy, yy) = key_years(x), key_years(y) + if sx != sy: + continue + if yx is None or yy is None or (yx[0] <= yy[1] and yy[0] <= yx[1]): + return True + return False + + +def family_of(entry: dict) -> set[str]: + return {FAMILY[key_years(key)[0]] for key in entry["unit_keys"]} + + +def make_entry(index: int, keys, path=("registration", "candidate")): + statuses = PATHS[path] + gate = f"gate_{index}" + return { + "id": f"H{index + 1:03d}", + "provenance": path[0], + "gate": gate, + "gate_identifiers": [gate], + "lane": "invented", + "status": statuses[-1], + "source": "invented source", + "units": {"years": [index]}, + "unit_keys": list(keys), + "evidence": [f"evidence {index}"], + "scored_evidence": ( + [{"path": run_path(f"s{index}"), "field": "gate", "equals": gate}] + if statuses[-1] == "used" + else [] + ), + "disclosed_overlaps": [], + "overlap_review": [], + "history": [ + {"date": f"2026-10-{20 + step:02d}", "status": s, "note": "n"} + for step, s in enumerate(statuses) + ], + } + + +def review(entry_id: str) -> dict: + return { + "id": entry_id, + "outcome": "disjoint", + "reason": REASON, + "evidence": ["https://example.org/catalog"], + } + + +def settle(entries: list[dict]) -> None: + """Disclose every shared unit both ways and review every unshared + same-family relation, as a careful registrant would.""" + live = [entry for entry in entries if entry["status"] != "withdrawn"] + for index, entry in enumerate(live): + for other in live[index + 1 :]: + if share_unit(entry, other): + if other["id"] not in entry["disclosed_overlaps"]: + entry["disclosed_overlaps"].append(other["id"]) + if entry["id"] not in other["disclosed_overlaps"]: + other["disclosed_overlaps"].append(entry["id"]) + for index, entry in enumerate(entries): + if entry["status"] == "withdrawn": + continue + reviewed = {item["id"] for item in entry["overlap_review"]} + for other in entries[:index]: + if ( + other["status"] != "withdrawn" + and family_of(entry) & family_of(other) + and not share_unit(entry, other) + and other["id"] not in entry["disclosed_overlaps"] + and other["id"] not in reviewed + ): + entry["overlap_review"].append(review(other["id"])) + + +def make_registry(entries: list[dict], complete=False) -> dict: + return { + "schema": registry.SCHEMA, + "title": "invented", + "purpose": "invented", + "rules": ["rule one", "the registry is incomplete"], + "status_values": {status: status for status in registry.STATUSES}, + "sources": copy.deepcopy(VOCABULARY), + "complete": complete, + "completion_record": "docs/design/inventory.json", + "entries": entries, + } + + +#: The invented scored runs, in place of the module's controlled list. +#: Without a repository no pin is checked. +SCORED_RUNS = { + run_path(f"s{index}"): (f"gate_{index}", "0" * 64) for index in range(12) +} + + +def authorize(current) -> dict: + """Authorize every well-formed historical import ``current`` holds as + it stands, for tests about other rules; the authorization rule itself + is tested against fixed authorizations.""" + entries = current.get("entries") if isinstance(current, dict) else None + imports = {} + for entry in entries if isinstance(entries, list) else []: + try: + if entry["provenance"] != "historical_import": + continue + imports[entry["id"]] = { + "gate": entry["gate"], + "unit_keys": tuple(entry["unit_keys"]), + "scored_runs": tuple( + item["path"] for item in entry["scored_evidence"] + ), + } + except (KeyError, TypeError): + continue + return imports + + +#: The legal transitions, specified independently of the module. +EXPECTED_TRANSITIONS = { + "candidate": {"frozen", "withdrawn"}, + "frozen": {"used", "withdrawn"}, + "used": set(), + "withdrawn": set(), +} + + +def validate( + current, repository=None, scored_runs=None, commit="HEAD", imports=None +): + return registry.validate( + current, + repository, + vocabulary=VOCABULARY, + commit=commit, + scored_runs=SCORED_RUNS if scored_runs is None else scored_runs, + imports=authorize(current) if imports is None else imports, + ) + + +entry_specs = st.lists( + st.tuples( + st.lists( + st.sampled_from(KEY_POOL), min_size=1, max_size=3, unique=True + ), + st.sampled_from(sorted(PATHS)), + ), + max_size=6, +) +registration_specs = st.lists( + st.lists(st.sampled_from(KEY_POOL), min_size=1, max_size=3, unique=True), + max_size=4, +) + + +def build(specs, complete=False) -> dict: + entries = [ + make_entry(i, keys, path) for i, (keys, path) in enumerate(specs) + ] + settle(entries) + return make_registry(entries, complete) + + +def test_the_field_lists_are_as_specified(): + assert registry.IMMUTABLE == EXPECTED_IMMUTABLE + assert registry.EXTENDING == EXPECTED_EXTENDING + assert set(EXPECTED_IMMUTABLE) | set(EXPECTED_EXTENDING) | { + "status", + } == set(registry.ENTRY_KEYS) + + +@settings(max_examples=200, deadline=None) +@given(entry_specs) +def test_settled_registries_are_valid(specs): + assert validate(build(specs)) == [] + + +@settings(max_examples=200, deadline=None) +@given(entry_specs) +def test_shared_units_match_the_independent_rule(specs): + current = build(specs) + entries = current["entries"] + for i, a in enumerate(entries): + for b in entries[i + 1 :]: + assert registry.entries_overlap(a, b) == share_unit(a, b) + + +@settings(max_examples=200, deadline=None) +@given(entry_specs, st.data()) +def test_a_one_way_disclosure_is_refused(specs, data): + current = build(specs) + disclosing = [e for e in current["entries"] if e["disclosed_overlaps"]] + if not disclosing: + return + data.draw(st.sampled_from(disclosing))["disclosed_overlaps"].pop() + problems = validate(current) + assert any( + "does not disclose it back" in p or "without disclosing" in p + for p in problems + ) + + +@settings(max_examples=200, deadline=None) +@given(entry_specs) +def test_a_silent_shared_unit_is_refused(specs): + current = build(specs) + for entry in current["entries"]: + entry["disclosed_overlaps"] = [] + entry["overlap_review"] = [] + settle_reviews_only(current["entries"]) + live = [e for e in current["entries"] if e["status"] != "withdrawn"] + shared = any( + share_unit(a, b) for i, a in enumerate(live) for b in live[i + 1 :] + ) + problems = validate(current) + assert any("without disclosing" in p for p in problems) == shared + + +def settle_reviews_only(entries): + for index, entry in enumerate(entries): + if entry["status"] == "withdrawn": + continue + for other in entries[:index]: + if ( + other["status"] != "withdrawn" + and family_of(entry) & family_of(other) + and not share_unit(entry, other) + ): + entry["overlap_review"].append(review(other["id"])) + + +@settings(max_examples=200, deadline=None) +@given(entry_specs) +def test_an_unreviewed_family_relation_is_refused(specs): + current = build(specs) + needed = any(entry["overlap_review"] for entry in current["entries"]) + for entry in current["entries"]: + entry["overlap_review"] = [] + problems = validate(current) + assert any("neither discloses nor reviews" in p for p in problems) == ( + needed + ) + + +def test_fixed_family_cases_follow_the_rule(): + # s_a and s_b share family f1 but no unit; s_c is another family. + same = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] + assert any( + "neither discloses nor reviews" in p + for p in validate(make_registry(copy.deepcopy(same))) + ) + reviewed = copy.deepcopy(same) + reviewed[1]["overlap_review"].append(review("H001")) + assert validate(make_registry(reviewed)) == [] + disclosed = copy.deepcopy(same) + disclosed[0]["disclosed_overlaps"].append("H002") + disclosed[1]["disclosed_overlaps"].append("H001") + assert validate(make_registry(disclosed)) == [] + other = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_c:2025"])] + assert validate(make_registry(other)) == [] + + +@pytest.mark.parametrize( + "change, message", + [ + ({"outcome": "unresolved"}, "is disclosed"), + ( + {"reason": "TODO: check later, once the inventory has landed."}, + "substantive", + ), + ({"reason": "short"}, "substantive"), + ({"evidence": []}, "cites nothing"), + ({"evidence": ["somewhere"]}, "not path:line or a URL"), + ({"evidence": ["docs/x.md:0"]}, "bad line range"), + ], +) +def test_a_review_needs_an_outcome_a_reason_and_citations(change, message): + entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] + entries[1]["overlap_review"].append(dict(review("H001"), **change)) + problems = validate(make_registry(entries)) + assert any(message in p for p in problems) + + +@settings(max_examples=200, deadline=None) +@given(entry_specs, registration_specs) +def test_appending_lone_candidates_keeps_every_rule(base_specs, more): + base = build(base_specs) + current = copy.deepcopy(base) + start = len(current["entries"]) + for offset, keys in enumerate(more): + current["entries"].append(make_entry(start + offset, keys)) + settle(current["entries"]) + assert validate(current) == [] + assert registry.append_only_problems(base, current) == [] + + +@settings(max_examples=100, deadline=None) +@given( + entry_specs, + st.sampled_from( + [p for p in PATHS if p[0] == "registration" and p[1] != "candidate"] + ), +) +def test_a_registration_cannot_arrive_already_frozen(base_specs, path): + base = build(base_specs) + current = copy.deepcopy(base) + current["entries"].append( + make_entry(len(current["entries"]), ["s_c:2099"], path) + ) + settle(current["entries"]) + assert validate(current) == [] + assert any( + "lone candidate" in p + for p in registry.append_only_problems(base, current) + ) + + +def test_imports_close_when_the_registry_is_complete(): + for complete in (False, True): + base = build([(["s_a:2020"], ("historical_import", "used"))], complete) + current = copy.deepcopy(base) + current["entries"].append( + make_entry(1, ["s_c:2099"], ("historical_import", "used")) + ) + problems = registry.append_only_problems(base, current) + assert bool(problems) == complete + + +IMMUTABLE_EDITS = { + "id": "H999", + "provenance": "registration", + "gate": "gate_other", + "lane": "other lane", + "source": "other source", + "units": {"years": [-1]}, + "unit_keys": ["s_c:2199"], +} + + +@settings(max_examples=150, deadline=None) +@given( + entry_specs.filter(bool), + st.sampled_from(sorted(IMMUTABLE_EDITS)), + st.data(), +) +def test_changing_an_earlier_entry_is_refused(specs, key, data): + base = build(specs) + current = copy.deepcopy(base) + entry = data.draw(st.sampled_from(current["entries"])) + if entry[key] == IMMUTABLE_EDITS[key]: + return + entry[key] = IMMUTABLE_EDITS[key] + assert registry.append_only_problems(base, current) + + +@settings(max_examples=150, deadline=None) +@given(entry_specs.filter(bool), st.data()) +def test_removing_or_reordering_entries_is_refused(specs, data): + base = build(specs) + removed = copy.deepcopy(base) + del removed["entries"][data.draw(st.integers(0, len(specs) - 1))] + assert registry.append_only_problems(base, removed) + if len(specs) > 1: + reordered = copy.deepcopy(base) + reordered["entries"].reverse() + assert registry.append_only_problems(base, reordered) + + +@settings(max_examples=150, deadline=None) +@given( + entry_specs.filter(bool), + st.sampled_from(EXPECTED_EXTENDING), + st.data(), +) +def test_extending_lists_never_shrink(specs, key, data): + base = build(specs) + current = copy.deepcopy(base) + index = data.draw(st.integers(0, len(specs) - 1)) + if not base["entries"][index][key]: + base["entries"][index][key] = ["something old"] + current["entries"][index][key] = ["something new"] + else: + current["entries"][index][key] = current["entries"][index][key][1:] + assert registry.append_only_problems(base, current) + + +def test_sources_rules_and_completeness_only_move_forward(): + base = make_registry([]) + for change in ("remove", "edit"): + edited = copy.deepcopy(base) + if change == "remove": + del edited["sources"]["s_b"] + else: + edited["sources"]["s_b"]["family"] = "f2" + assert registry.append_only_problems(base, edited) + added = copy.deepcopy(base) + added["sources"]["s_d"] = { + "family": "f3", + "title": "D", + "catalog": "cat d", + "key_year": "y", + } + assert registry.append_only_problems(base, added) == [] + trimmed = copy.deepcopy(base) + trimmed["rules"] = trimmed["rules"][1:] + assert registry.append_only_problems(base, trimmed) + done = copy.deepcopy(base) + done["complete"] = True + assert registry.append_only_problems(base, done) == [] + assert registry.append_only_problems(done, base) + renamed = copy.deepcopy(base) + renamed["schema"] = "track_b_holdout_registry.v2" + assert registry.append_only_problems(base, renamed) + + +def test_the_registry_must_carry_the_controlled_vocabulary(): + current = make_registry([]) + current["sources"]["s_d"] = dict(VOCABULARY["s_c"], catalog="cat d") + assert any("controlled vocabulary" in p for p in validate(current)) + aliased = dict(VOCABULARY, s_d=dict(VOCABULARY["s_a"], catalog=" CAT A")) + problems = registry.validate( + dict(current, sources=aliased), vocabulary=aliased + ) + assert any("alias" in p for p in problems) + + +@given( + st.sampled_from(sorted(VOCABULARY)), + st.integers(1900, 2100), + st.integers(0, 5), + st.integers(1900, 2100), + st.integers(0, 5), + st.booleans(), +) +def test_keys_overlap_on_source_and_intersecting_years(s, y1, n1, y2, n2, x): + a = f"{s}:{y1}-{y1 + n1}:alpha" + b = f"{s}:{y2}-{y2 + n2}" + (":beta" if x else "") + intersect = y1 <= y2 + n2 and y2 <= y1 + n1 + assert registry.keys_overlap(a, b) == intersect + assert registry.keys_overlap(b, a) == intersect + assert registry.keys_overlap(a, f"{s}:undated") + other = "s_c" if s != "s_c" else "s_a" + assert not registry.keys_overlap(a, f"{other}:{y1}") + + +@pytest.mark.parametrize( + "key", + [ + "s_a", + "s_a:20", + "s_a:2022-2020", + "s_a:2020:Upper", + "S_A:2020", + "s_a:2020::x", + "s_a:dated", + 7, + ], +) +def test_malformed_unit_keys_are_refused(key): + entry = make_entry(0, ["s_a:2020"]) + entry["unit_keys"] = [key] + assert validate(make_registry([entry])) + + +def test_a_key_naming_no_source_is_refused(): + entry = make_entry(0, ["s_z:2020"]) + problems = validate(make_registry([entry])) + assert any("names no source" in p for p in problems) + + +@pytest.mark.parametrize( + "provenance, statuses, last, message", + [ + ("registration", ["frozen"], "frozen", "starts as one of"), + ("historical_import", ["candidate"], "candidate", "starts as one of"), + ("historical_import", ["frozen"], "frozen", "starts as one of"), + ("registration", ["candidate", "used"], "used", "not a legal"), + ("historical_import", ["used", "frozen"], "frozen", "not a legal"), + ("registration", ["candidate", "frozen"], "candidate", "last status"), + ], +) +def test_status_follows_a_legal_history(provenance, statuses, last, message): + entry = make_entry(0, ["s_a:2020"]) + entry["provenance"] = provenance + entry["history"] = [ + {"date": "2026-10-21", "status": s, "note": "n"} for s in statuses + ] + entry["status"] = last + if last == "used": + entry["scored_evidence"] = [ + {"path": run_path("s"), "field": "gate", "equals": "gate_0"} + ] + assert any(message in p for p in validate(make_registry([entry]))) + + +@pytest.mark.parametrize("date", ["2026-10-01", "2026-99-99", "yesterday"]) +def test_history_dates_are_real_and_never_go_back(date): + entry = make_entry(0, ["s_a:2020"], ("registration", "frozen")) + entry["history"][1]["date"] = date + assert validate(make_registry([entry])) + + +def test_repeated_ids_and_shared_gate_identifiers_are_refused(): + current = build([(["s_a:2020"], ("registration", "candidate"))] * 2) + current["entries"][1]["id"] = "H001" + assert any("ids repeat" in p for p in validate(current)) + current = build([(["s_a:2020"], ("registration", "candidate"))] * 2) + current["entries"][1]["gate_identifiers"].append("gate_0") + assert any("already identifies" in p for p in validate(current)) + + +@pytest.mark.parametrize( + "change, message", + [ + ({"equals": "gate_other"}, "does not identify this gate"), + ({"field": "title"}, "not an identity field"), + ({"path": "docs/design/registry.json"}, "not a listed scored run"), + ({"path": "../outside.json"}, "not a listed scored run"), + ({"path": run_path("s1")}, "'gate_1''s scored run"), + ], +) +def test_scored_evidence_must_be_a_run_artifact_naming_this_gate( + change, message +): + entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + entry["scored_evidence"][0].update(change) + assert any(message in p for p in validate(make_registry([entry]))) + entry["scored_evidence"] = [] + problems = validate(make_registry([entry])) + assert any("needs scored_evidence" in p for p in problems) + + +def test_the_transitions_are_as_specified(): + assert { + status: set(following) + for status, following in registry.TRANSITIONS.items() + } == EXPECTED_TRANSITIONS + + +@pytest.mark.parametrize( + "statuses", + [ + ["candidate", "frozen", "used", "withdrawn"], + ["candidate", "withdrawn", "frozen"], + ["candidate", "frozen", "used", "frozen"], + ["candidate", "frozen", "frozen"], + ], +) +def test_terminal_and_repeated_statuses_are_refused(statuses): + entry = make_entry(0, ["s_a:2020"]) + entry["history"] = [ + {"date": "2026-10-21", "status": s, "note": "n"} for s in statuses + ] + entry["status"] = statuses[-1] + entry["scored_evidence"] = [ + {"path": run_path("s0"), "field": "gate", "equals": "gate_0"} + ] + assert any( + "not a legal transition" in p for p in validate(make_registry([entry])) + ) + + +#: One fixed authorization: H001 imports gate_0's s_a:2020 with run s0. +AUTHORIZED = { + "H001": { + "gate": "gate_0", + "unit_keys": ("s_a:2020",), + "scored_runs": (run_path("s0"),), + } +} + + +def import_with(change) -> dict: + """gate_0's authorized import, then ``change`` applied to its entry.""" + entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + change(entry) + return make_registry([entry]) + + +def _rename(entry): + entry["id"] = "H002" + + +def _regate(entry): + entry.update(gate="gate_1", gate_identifiers=["gate_1"]) + entry["scored_evidence"][0].update(path=run_path("s1"), equals="gate_1") + + +def _more_units(entry): + entry["unit_keys"].append("s_c:2099") + + +def _other_units(entry): + entry["unit_keys"] = ["s_c:2099"] + + +def _more_runs(entry): + entry["scored_evidence"].append(dict(entry["scored_evidence"][0])) + + +@pytest.mark.parametrize( + "change, message", + [ + (_rename, "not an authorized historical import"), + (_regate, "is authorized for 'gate_0'"), + (_more_units, "unit keys differ"), + (_other_units, "unit keys differ"), + (_more_runs, "differs from its authorized runs"), + ], +) +def test_imports_must_match_their_authorization(change, message): + assert validate(import_with(lambda entry: None), imports=AUTHORIZED) == [] + problems = validate(import_with(change), imports=AUTHORIZED) + assert any(message in p for p in problems), problems + + +def test_an_authorization_admits_one_import_only(): + first = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + again = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + again["id"] = "H002" + entries = [first, again] + settle(entries) + problems = validate(make_registry(entries), imports=AUTHORIZED) + assert any( + "H002: is not an authorized historical import" in p for p in problems + ) + + +def test_a_review_must_name_an_earlier_entry(): + entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_c:2025"])] + entries[0]["overlap_review"].append(review("H002")) + problems = validate(make_registry(entries)) + assert any("not an earlier entry" in p for p in problems) + + +@pytest.mark.parametrize( + "path, value", + [ + ((), None), + (("title",), None), + (("purpose",), []), + (("rules",), []), + (("status_values", "used"), 3), + (("sources", "s_a", "family"), ""), + (("complete",), "no"), + (("entries", 0, "scored_evidence"), 7), + (("entries", 0, "history"), "used"), + (("entries", 0, "history", 0), "used"), + (("entries", 0, "units"), {}), + (("entries", 0, "gate"), None), + (("entries", 0, "overlap_review"), [{"id": "H001"}]), + (("entries", 0, "disclosed_overlaps"), [3]), + ], +) +def test_malformed_fields_are_reported_not_raised(path, value): + current = build([(["s_a:2020"], ("historical_import", "used"))]) + if not path: + current = value + else: + target = current + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + problems = validate(current) + assert isinstance(problems, list) and problems + + +json_values = st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=8), + lambda children: st.lists(children, max_size=4) + | st.dictionaries(st.text(max_size=8), children, max_size=4), + max_leaves=20, +) + + +@settings(max_examples=200, deadline=None) +@given( + st.sampled_from(sorted(registry.ENTRY_KEYS) + ["entries"]), + json_values, + st.booleans(), +) +def test_any_json_in_any_field_is_reported_not_raised(key, value, append): + base = build([(["s_a:2020"], ("historical_import", "used"))]) + current = copy.deepcopy(base) + if key == "entries": + current["entries"] = value + elif append: + added = make_entry(1, ["s_c:2099"]) + added[key] = value + current["entries"].append(added) + else: + current["entries"][0][key] = value + assert isinstance(validate(current), list) + assert isinstance(registry.append_only_problems(base, current), list) + + +# Committed trees, scored runs and base commits, through git ---------------- + + +def git(repository: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def init_repository(tmp_path: Path) -> Path: + repository = tmp_path / "repo" + repository.mkdir() + git(repository, "init", "-q", "-b", "master") + git(repository, "config", "user.email", "t@example.com") + git(repository, "config", "user.name", "t") + write( + repository, + registry.GATES, + "gates:\n gate_0:\n id: zero_gate\n" + " registration_id: reg-zero\n", + ) + commit_all(repository, "start") + return repository + + +def write(repository: Path, relative: str, document) -> None: + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(document, bytes): + path.write_bytes(document) + return + text = document if isinstance(document, str) else json.dumps(document) + path.write_text(text, encoding="utf-8") + + +def commit_all(repository: Path, message: str) -> str: + git(repository, "add", "-A") + git(repository, "commit", "-q", "--allow-empty", "-m", message) + return git(repository, "rev-parse", "HEAD").strip() + + +def pin(document, gate: str = "gate_0") -> dict: + """The scored-run list for one artifact holding ``document``.""" + text = json.dumps(document).encode("utf-8") + return {run_path("s0"): (gate, hashlib.sha256(text).hexdigest())} + + +@pytest.fixture +def no_ci_env(monkeypatch): + for name in ("GITHUB_BASE_REF", "GITHUB_EVENT_NAME", "GITHUB_EVENT_PATH"): + monkeypatch.delenv(name, raising=False) + + +SCORE = {"gate": "gate_0"} + + +def imported(document=SCORE) -> dict: + entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + entry["gate_identifiers"] = ["gate_0", "zero_gate", "reg-zero"] + return make_registry([entry]) + + +def test_scored_evidence_is_read_from_the_committed_tree(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = imported() + evidence = current["entries"][0]["scored_evidence"][0] + runs = pin(SCORE) + write(repository, evidence["path"], SCORE) + assert any( + "not a committed file" in p + for p in validate(current, repository, runs) + ) + git(repository, "add", "-A") + assert any( + "not a committed file" in p + for p in validate(current, repository, runs) + ) + git(repository, "commit", "-q", "-m", "score") + assert validate(current, repository, runs) == [] + write(repository, evidence["path"], {"gate": "gate_9"}) + assert validate(current, repository, runs) == [] + for field, equals in ( + ("registration_id", "reg-zero"), + ("gate", "zero_gate"), + ): + evidence.update(field=field, equals=equals) + assert any( + "does not have" in p for p in validate(current, repository, runs) + ) + + +def test_a_scored_run_must_keep_its_pinned_bytes(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = imported() + path = current["entries"][0]["scored_evidence"][0]["path"] + write(repository, path, {"gate": "gate_0", "floor": True}) + commit_all(repository, "a floor, not the scored run") + problems = validate(current, repository, pin(SCORE)) + assert any("differs from the listed scored run" in p for p in problems) + + +def test_identifiers_must_be_authoritative(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = imported() + entry = current["entries"][0] + document = {"gate": "x"} + write(repository, entry["scored_evidence"][0]["path"], document) + commit_all(repository, "score") + entry["gate_identifiers"].append("x") + entry["scored_evidence"][0].update(field="gate", equals="x") + assert any( + "does not identify this gate" in p + for p in validate(current, repository, pin(document)) + ) + entry["gate"] = "gate_unknown" + entry["gate_identifiers"] = ["gate_unknown"] + entry["scored_evidence"][0]["equals"] = "gate_unknown" + runs = pin(document, "gate_unknown") + assert any( + "neither a gates.yaml gate" in p + for p in validate(current, repository, runs) + ) + + +def test_a_symlinked_or_non_json_artifact_is_refused(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = imported() + path = current["entries"][0]["scored_evidence"][0]["path"] + outside = tmp_path / "outside.json" + outside.write_text(json.dumps(SCORE), encoding="utf-8") + link = repository / path + link.parent.mkdir(parents=True) + link.symlink_to(outside) + commit_all(repository, "link") + assert any( + "not a committed file" in p + for p in validate(current, repository, pin(SCORE)) + ) + link.unlink() + write(repository, path, b"\xff\xfe not json") + commit_all(repository, "bytes") + runs = {path: ("gate_0", hashlib.sha256(b"\xff\xfe not json").hexdigest())} + assert any( + "does not have" in p for p in validate(current, repository, runs) + ) + + +def registered(statuses) -> dict: + """A registration of gate_0 at ``statuses``, citing s0 once used.""" + entry = make_entry(0, ["s_a:2020"]) + entry["gate_identifiers"] = ["gate_0", "zero_gate", "reg-zero"] + entry["history"] = [ + {"date": f"2026-10-{21 + step:02d}", "status": s, "note": "n"} + for step, s in enumerate(statuses) + ] + entry["status"] = statuses[-1] + if statuses[-1] == "used": + entry["scored_evidence"] = [ + {"path": run_path("s0"), "field": "gate", "equals": "gate_0"} + ] + return make_registry([entry]) + + +def commit_registry(repository: Path, current: dict, message: str) -> str: + write(repository, registry.REGISTRY, current) + return commit_all(repository, message) + + +@pytest.mark.parametrize("registered_first", [True, False]) +def test_a_registration_must_precede_its_scored_run( + tmp_path, no_ci_env, registered_first +): + repository = init_repository(tmp_path) + if not registered_first: + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score first") + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + if registered_first: + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + if registered_first: + assert problems == [] + else: + assert any("already committed" in p for p in problems) + + +def git_rename(repository: Path, old: str, new: str) -> None: + (repository / new).parent.mkdir(parents=True, exist_ok=True) + git(repository, "mv", old, new) + + +@pytest.mark.parametrize( + "scenario", ["moved", "deleted_and_restored", "same_commit", "side_branch"] +) +def test_a_score_committed_anywhere_before_registration_is_refused( + tmp_path, no_ci_env, scenario +): + """The score's bytes were committed before the registration in every + scenario: at another path then moved, at its path then deleted and + restored, in the registering commit itself, or on the base branch + while the registration waited on a side branch merged afterwards.""" + repository = init_repository(tmp_path) + listed, elsewhere = run_path("s0"), run_path("original") + if scenario == "moved": + write(repository, elsewhere, SCORE) + commit_all(repository, "score at another path") + elif scenario == "deleted_and_restored": + write(repository, listed, SCORE) + commit_all(repository, "score") + (repository / listed).unlink() + commit_all(repository, "score deleted") + if scenario == "same_commit": + write(repository, listed, SCORE) + commit_registry(repository, registered(["candidate"]), "both") + elif scenario == "side_branch": + git(repository, "checkout", "-q", "-b", "side") + commit_registry(repository, registered(["candidate"]), "register") + git(repository, "checkout", "-q", "master") + write(repository, listed, SCORE) + commit_all(repository, "score") + git(repository, "merge", "-q", "--no-ff", "-m", "merge side", "side") + else: + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + if scenario == "moved": + git_rename(repository, elsewhere, listed) + commit_all(repository, "moved to its listed path") + elif scenario == "deleted_and_restored": + write(repository, listed, SCORE) + commit_all(repository, "score restored") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + assert any("already committed" in p for p in problems), problems + + +def test_a_shallow_clone_cannot_prove_a_score_was_not_committed( + tmp_path, no_ci_env +): + """A shallow clone that keeps the registry's history but not the + score's first commit must not read an empty search as absence.""" + repository = init_repository(tmp_path) + listed = run_path("s0") + write(repository, listed, SCORE) + commit_all(repository, "score") + (repository / listed).unlink() + commit_all(repository, "score deleted") + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, listed, SCORE) + commit_all(repository, "score restored") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + shallow = tmp_path / "shallow" + subprocess.run( + ["git", "clone", "-q", "--depth", "5", repository.as_uri(), shallow], + check=True, + capture_output=True, + ) + assert git(shallow, "rev-parse", "--is-shallow-repository").strip() == ( + "true" + ) + assert any( + "already committed" in p + for p in validate(used, repository, pin(SCORE)) + ) + problems = validate(used, shallow, pin(SCORE)) + assert any("cannot tell whether" in p for p in problems), problems + + +def test_a_replacement_ref_cannot_hide_a_committed_score(tmp_path, no_ci_env): + """`git replace` can graft a parentless copy over a commit and hide + its ancestors from ordinary log walks; the search ignores it.""" + repository = init_repository(tmp_path) + listed = run_path("s0") + write(repository, listed, SCORE) + commit_all(repository, "score") + (repository / listed).unlink() + deleted = commit_all(repository, "score deleted") + orphan = git( + repository, "commit-tree", f"{deleted}^{{tree}}", "-m", "orphan" + ).strip() + git(repository, "replace", deleted, orphan) + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, listed, SCORE) + commit_all(repository, "score restored") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + assert any("already committed" in p for p in problems), problems + + +def test_a_score_added_only_in_a_merge_resolution_is_refused( + tmp_path, no_ci_env +): + """A score that enters only through a merge commit's own resolution, + before the registration, was committed before it.""" + repository = init_repository(tmp_path) + git(repository, "checkout", "-q", "-b", "side") + write(repository, "docs/side.md", "side\n") + commit_all(repository, "side work") + git(repository, "checkout", "-q", "master") + write(repository, "docs/main.md", "main\n") + commit_all(repository, "main work") + git(repository, "merge", "-q", "--no-ff", "--no-commit", "side") + write(repository, run_path("s0"), SCORE) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "merge, adding the score") + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + assert any("already committed" in p for p in problems), problems + + +def test_subprocess_failures_are_reported_not_raised( + tmp_path, no_ci_env, monkeypatch +): + repository = init_repository(tmp_path) + huge = "a" * 3_000_000 + current = registered(["candidate"]) + current.update(completion_record=huge, complete=True) + assert isinstance(validate(current, repository), list) + for commit in (None, huge): + with pytest.raises(LookupError): + registry.registry_at(repository, commit) + with pytest.raises(LookupError): + registry.history_problems(repository, huge, imports={}) + monkeypatch.setenv("PATH", str(tmp_path / "no-git-here")) + with pytest.raises(LookupError): + registry.registry_at(repository, "HEAD") + assert validate(imported(), repository, pin(SCORE)) + + +@pytest.mark.parametrize( + "imports, message", + [ + (None, "must be a mapping"), + ({"H001": {}}, "needs a gate"), + ( + {"H001": {"gate": "g", "unit_keys": [1], "scored_runs": []}}, + "needs", + ), + ], +) +def test_malformed_authorizations_are_reported_not_raised(imports, message): + problems = registry.validate( + imported(), vocabulary=VOCABULARY, imports=imports + ) + assert any(message in p for p in problems), problems + + +def test_sha256_object_ids_bound_histories(tmp_path, no_ci_env, monkeypatch): + repository = tmp_path / "repo256" + repository.mkdir() + git(repository, "init", "-q", "-b", "master", "--object-format=sha256") + git(repository, "config", "user.email", "t@example.com") + git(repository, "config", "user.name", "t") + first = registered(["candidate"]) + before = commit_registry(repository, first, "registry") + assert len(before) == 64 + changed = copy.deepcopy(first) + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "an immutable key changes") + commit_registry(repository, first, "and is restored") + commit_all(repository, "an unchanged tip") + event = tmp_path / "event.json" + event.write_text( + json.dumps( + {"before": before, "pull_request": {"head": {"sha": before}}} + ), + encoding="utf-8", + ) + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + assert registry.base_commit(repository) == before + assert registry.history_problems( + repository, registry.base_commit(repository), imports={} + ) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + assert registry.pull_request_head() == before + + +def test_an_unanswerable_timing_check_fails_closed( + tmp_path, no_ci_env, monkeypatch +): + repository = init_repository(tmp_path) + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + assert validate(used, repository, pin(SCORE)) == [] + monkeypatch.setattr( + registry.Committed, "committed_by", lambda self, content, at: None + ) + problems = validate(used, repository, pin(SCORE)) + assert any("cannot tell whether" in p for p in problems) + + +@pytest.mark.parametrize( + "failure", + [ + OSError(errno.EMFILE, "Too many open files"), + FileNotFoundError(errno.ENOENT, "git"), + TypeError("expected str, bytes or os.PathLike object"), + ValueError("embedded null byte"), + ], +) +def test_a_failed_object_hash_fails_closed( + tmp_path, no_ci_env, monkeypatch, failure +): + repository = init_repository(tmp_path) + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + assert validate(used, repository, pin(SCORE)) == [] + run = subprocess.run + + def failing(command, *args, **kwargs): + if "hash-object" in command: + raise failure + return run(command, *args, **kwargs) + + monkeypatch.setattr(registry.subprocess, "run", failing) + committed = registry.Committed(repository) + assert committed.committed_by(json.dumps(SCORE).encode(), "HEAD") is None + problems = validate(used, repository, pin(SCORE)) + assert any("cannot tell whether" in p for p in problems), problems + + +def test_a_used_registration_must_already_be_registered(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + problems = validate(used, repository, pin(SCORE)) + assert any("before it is registered" in p for p in problems) + + +def test_each_snapshot_is_validated_against_its_own_tree(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = git(repository, "rev-parse", "HEAD").strip() + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used before its score is committed") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "the score, one commit late") + runs = pin(SCORE) + assert validate(used, repository, runs) == [] + problems = registry.history_problems( + repository, base, scored_runs=runs, imports={} + ) + assert any("not a committed file" in p for p in problems) + + +def test_a_new_registry_is_admitted_against_an_empty_one(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = git(repository, "rev-parse", "HEAD").strip() + commit_registry( + repository, registered(["candidate", "frozen"]), "arrives frozen" + ) + problems = registry.history_problems(repository, base, imports={}) + assert any("lone candidate" in p for p in problems) + + +def test_a_clean_history_has_no_problems(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = git(repository, "rev-parse", "HEAD").strip() + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + commit_registry( + repository, registered(["candidate", "frozen", "used"]), "used" + ) + assert ( + registry.history_problems( + repository, + base, + scored_runs=pin(SCORE), + imports={}, + ) + == [] + ) + + +@pytest.mark.parametrize("undo", ["append_then_remove", "edit_then_restore"]) +@pytest.mark.parametrize("full", [True, False]) +def test_a_run_of_commits_cannot_hide_a_change_it_undoes( + tmp_path, no_ci_env, undo, full +): + repository = init_repository(tmp_path) + first = registered(["candidate"]) + base = commit_registry(repository, first, "registry") + changed = copy.deepcopy(first) + if undo == "append_then_remove": + changed["entries"].append(make_entry(1, ["s_c:2099"])) + else: + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "change") + commit_registry(repository, first, "undo") + assert registry.append_only_problems(first, first) == [] + assert registry.history_problems(repository, base, full=full, imports={}) + + +def test_a_pull_request_head_is_read_from_its_event(tmp_path, monkeypatch): + event = tmp_path / "event.json" + sha = "a" * 40 + event.write_text( + json.dumps({"pull_request": {"head": {"sha": sha}}}), encoding="utf-8" + ) + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + assert registry.pull_request_head() == sha + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + assert registry.pull_request_head() is None + + +@pytest.mark.parametrize( + "payload", + [ + None, + "not json", + "[]", + json.dumps({}), + json.dumps({"pull_request": None}), + json.dumps({"pull_request": {"head": "abc"}}), + json.dumps({"pull_request": {"head": {"sha": "HEAD"}}}), + ], +) +def test_an_unreadable_pull_request_event_raises( + tmp_path, monkeypatch, payload +): + event = tmp_path / "event.json" + if payload is not None: + event.write_text(payload, encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + with pytest.raises(LookupError): + registry.pull_request_head() + monkeypatch.setenv("GITHUB_EVENT_NAME", "workflow_dispatch") + assert registry.pull_request_head() is None + + +def test_malformed_snapshots_are_reported_not_raised(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = commit_registry(repository, registered(["candidate"]), "registry") + for bad in ("not json", [], {"entries": None}): + write( + repository, + registry.REGISTRY, + bad if isinstance(bad, str) else json.dumps(bad), + ) + commit_all(repository, "a malformed snapshot") + commit_registry(repository, registered(["candidate"]), "repaired") + problems = registry.history_problems(repository, base, imports={}) + assert len(problems) >= 3 + assert registry.append_only_problems(registered(["candidate"]), None) + + +def test_malformed_predecessors_and_parser_limits_are_reported( + tmp_path, no_ci_env +): + no_id = {"entries": [{}]} + assert registry.append_only_problems( + no_id, {"entries": [{"units": {"years": [2020]}}]} + ) + repository = init_repository(tmp_path) + base = commit_registry(repository, no_id, "a malformed predecessor") + commit_registry(repository, registered(["candidate"]), "repaired") + for full in (True, False): + problems = registry.history_problems( + repository, base, full=full, imports={} + ) + assert isinstance(problems, list) + huge = "9" * 5000 + write(repository, registry.GATES, f"gates:\n gate_0:\n id: {huge}\n") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "an identifier past Python's integer limit") + assert isinstance(validate(imported(), repository, pin(SCORE)), list) + write(repository, registry.REGISTRY, '{"entries": [' + huge + "]}") + huge_json = commit_all(repository, "a registry past the integer limit") + with pytest.raises(ValueError, match="is not JSON"): + registry.registry_at(repository, huge_json) + nul = "HEAD\x00" + assert validate(imported(), repository, pin(SCORE), commit=nul) + with pytest.raises(LookupError): + registry.registry_at(repository, nul) + with pytest.raises(LookupError): + registry.history_problems(repository, nul, imports={}) + + +def test_review_citations_must_name_committed_text_lines(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + write(repository, "docs/note.md", "one\ntwo\nthree\n") + write(repository, "docs/binary.md", b"\xff\xfe\x00\x01") + commit_all(repository, "notes") + entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] + entries[1]["overlap_review"].append(review("H001")) + current = make_registry(entries) + for citation, ok in ( + ("docs/note.md:2", True), + ("docs/note.md:2-3", True), + ("docs/note.md:4", False), + ("docs/missing.md:1", False), + ("docs/binary.md:1", False), + ("docs/note.md:12345678", False), + ("docs/../docs/note.md:2", False), + ): + entries[1]["overlap_review"][0]["evidence"] = [citation] + assert (validate(current, repository) == []) == ok, citation + + +def test_complete_needs_its_committed_record(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = make_registry([], complete=True) + assert any("not committed" in p for p in validate(current, repository)) + write(repository, current["completion_record"], {}) + commit_all(repository, "inventory") + assert validate(current, repository) == [] + for path in ("docs/a\x00b.json", "docs/../docs/design/inventory.json"): + current["completion_record"] = path + problems = validate(current, repository) + assert any("is not a path" in p for p in problems), path + + +def test_a_pull_request_compares_with_its_merge_base(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + git(repository, "checkout", "-q", "-b", "feature") + write(repository, "notes.txt", "a commit of the branch's own") + commit_all(repository, "feature work") + assert registry.base_registry(repository) is None + git(repository, "checkout", "-q", "master") + first = registered(["candidate"]) + commit_registry(repository, first, "registry") + git(repository, "checkout", "-q", "-b", "feature-2") + second = copy.deepcopy(first) + second["entries"].append(make_entry(1, ["s_c:2099"])) + commit_registry(repository, second, "append") + assert registry.base_registry(repository) == first + + +def test_the_base_branch_compares_with_its_parent(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + first = registered(["candidate"]) + commit_registry(repository, first, "registry") + emptied = copy.deepcopy(first) + emptied["entries"] = [] + commit_registry(repository, emptied, "delete H001") + base = registry.base_registry(repository) + assert base == first + assert registry.append_only_problems(base, emptied) + + +def test_a_push_compares_with_its_before_commit( + tmp_path, no_ci_env, monkeypatch +): + repository = init_repository(tmp_path) + first = registered(["candidate"]) + before = commit_registry(repository, first, "registry") + middle = copy.deepcopy(first) + middle["entries"].append(make_entry(1, ["s_c:2099"])) + commit_registry(repository, middle, "append") + emptied = copy.deepcopy(first) + emptied["entries"] = [] + commit_registry(repository, emptied, "delete everything") + event = tmp_path / "event.json" + event.write_text(json.dumps({"before": before}), encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + assert registry.base_commit(repository) == before + assert registry.append_only_problems( + registry.base_registry(repository), emptied + ) + event.write_text(json.dumps({"before": "0" * 40}), encoding="utf-8") + assert registry.base_registry(repository) == middle + + +@pytest.mark.parametrize( + "payload", + [ + None, + "not json", + "[]", + json.dumps({}), + json.dumps({"before": None}), + json.dumps({"before": "HEAD"}), + json.dumps({"before": "b" * 40}), + ], +) +def test_a_push_without_a_resolvable_before_raises( + tmp_path, no_ci_env, monkeypatch, payload +): + repository = init_repository(tmp_path) + commit_registry(repository, registered(["candidate"]), "registry") + event = tmp_path / "event.json" + if payload is not None: + event.write_text(payload, encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + with pytest.raises(LookupError): + registry.base_commit(repository) + with pytest.raises(LookupError): + registry.base_registry(repository) + + +@pytest.mark.parametrize("errno_code", [errno.EAGAIN, errno.EMFILE]) +def test_a_transient_failure_at_the_push_boundary_raises( + tmp_path, no_ci_env, monkeypatch, errno_code +): + repository = init_repository(tmp_path) + first = registered(["candidate"]) + before = commit_registry(repository, first, "registry") + changed = copy.deepcopy(first) + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "an immutable key changes") + commit_registry(repository, first, "and is restored") + commit_all(repository, "an unchanged tip") + event = tmp_path / "event.json" + event.write_text(json.dumps({"before": before}), encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + assert registry.base_commit(repository) == before + assert registry.history_problems( + repository, registry.base_commit(repository), imports={} + ) + run = subprocess.run + + def flaky(command, *args, **kwargs): + if f"{before}^{{commit}}" in command: + raise OSError(errno_code, os.strerror(errno_code)) + return run(command, *args, **kwargs) + + monkeypatch.setattr(registry.subprocess, "run", flaky) + with pytest.raises(LookupError): + registry.base_commit(repository) + + +def test_no_base_raises_rather_than_passing(tmp_path, monkeypatch): + monkeypatch.delenv("GITHUB_EVENT_NAME", raising=False) + monkeypatch.setenv("GITHUB_BASE_REF", "no-such-branch") + repository = init_repository(tmp_path) + with pytest.raises(LookupError): + registry.base_registry(repository) + monkeypatch.delenv("GITHUB_BASE_REF") + with pytest.raises(LookupError): + registry.base_registry(repository) diff --git a/tests/test_track_b_holdout_registry_committed.py b/tests/test_track_b_holdout_registry_committed.py new file mode 100644 index 00000000..a7e8b6e5 --- /dev/null +++ b/tests/test_track_b_holdout_registry_committed.py @@ -0,0 +1,230 @@ +"""The committed held-out data registry keeps its rules on every commit. + +``docs/design/track_b_holdout_registry.json`` is the single registry of +the data that Track B's gates and the open-population lane's gates hold +out (dynamics coordinator, 2026-10-21). These tests check the committed +registry against the committed tree, including its scored-run artifacts +under ``runs/``, against its base commit, and between every first-parent +commit since that base. The rules themselves are exercised on invented +registries in ``test_track_b_holdout_registry.py``. +""" + +from __future__ import annotations + +import hashlib +import os +from pathlib import Path + +import pytest + +from populace_dynamics.track_b import holdout_registry as registry + +ROOT = Path(__file__).resolve().parents[1] + +#: The seeded entries' unit keys, specified here independently. They are +#: immutable once merged: PSID by interview wave, EPUF by the vintage in +#: its title, publications by edition. +EXPECTED_SEED_KEYS = { + "H001": ["psid:2015-2023:held_out_window"], + "H002": ["psid:2013:fam2013er", "psid:2015:fam2015er"], + "H003": ["ssa_epuf:2006:test_partition"], + "H004": [ + "psid:1982:employment_status", + "psid:1983:employment_status", + "psid:1993-1997:employment_status", + "psid:1999-2023:employment_status", + "ssa_di_asr:2023", + ], + "H005": [ + "ssa_supplement:2023:6.b5.1", + "ssa_supplement:2023:6.b4", + "ssa_supplement:2023:6.b3", + "ssa_di_asr:2023:table_19", + "census_cps_tables:2023:hh-4", + "census_cps_tables:2023:ad-3", + "populace_us_frame:2024:certified", + "mermin_smith_anchors:undated", + ], +} +#: Every disclosed pair: four shared units and two unresolved relations +#: between EPUF and the 2023 DI report and Supplement tables. +EXPECTED_DISCLOSURES = { + ("H001", "H002"), + ("H001", "H004"), + ("H002", "H004"), + ("H003", "H004"), + ("H003", "H005"), + ("H004", "H005"), +} +EXPECTED_FAMILIES = { + "psid": "psid", + "ssa_epuf": "ssa_administrative", + "ssa_supplement": "ssa_administrative", + "ssa_di_asr": "ssa_administrative", + "census_cps_tables": "us_cross_section", + "populace_us_frame": "us_cross_section", + "mermin_smith_anchors": "literature", +} +EXPECTED_OUTSIDE_GATES_YAML = {"B2": ("B2", "6252388925")} +#: Every listed scored run and its gate, specified here independently. +EXPECTED_SCORED_RUN_GATES = { + "runs/gate_m6_candidate1_v1.json": "gate_m6", + "runs/gate_m6_candidate2_v1.json": "gate_m6", + "runs/gate_m6_candidate3_v1.json": "gate_m6", + "runs/track_b_b2_score.json": "B2", + "runs/epuf_fill_gate_test_v1.json": "gate_epuf_fill", + "runs/gate_m4_hazard_v1.json": "gate_m4", + "runs/gate_w1_candidate1_v1.json": "gate_w1", + "runs/gate_w1_candidate2_v1.json": "gate_w1", + "runs/gate_w1_candidate3_v1.json": "gate_w1", + "runs/gate_w1_candidate4_v1.json": "gate_w1", +} +#: The authorized historical imports: each id's gate. Its unit keys are +#: the seeded keys above, and its scored runs are all of its gate's listed +#: runs. +EXPECTED_IMPORT_GATES = { + "H001": "gate_m6", + "H002": "B2", + "H003": "gate_epuf_fill", + "H004": "gate_m4", + "H005": "gate_w1", +} + + +def committed() -> dict: + return registry.load(ROOT) + + +def by_id(current: dict) -> dict: + return {entry["id"]: entry for entry in current["entries"]} + + +def test_the_committed_registry_keeps_its_rules(): + assert registry.validate(committed(), ROOT) == [] + + +def resolved_base(): + try: + return registry.base_commit(ROOT) + except LookupError as error: + if os.environ.get("GITHUB_ACTIONS") == "true": + pytest.fail(f"CI must resolve the base commit: {error}") + pytest.skip(f"no base commit here: {error}") + + +def test_the_committed_registry_only_appends_to_its_base(): + base_commit = resolved_base() + base = registry.registry_at(ROOT, base_commit) + current = committed() + if base is None: + assert ( + registry.append_only_problems( + registry.empty_like(current), current + ) + == [] + ) + else: + assert registry.validate(base, vocabulary=base["sources"]) == [] + assert registry.append_only_problems(base, current) == [] + + +def test_a_pull_requests_own_commits_append_only(): + head = registry.pull_request_head() + if head is None: + pytest.skip("not a pull_request event") + base_commit = resolved_base() + if registry.registry_at(ROOT, base_commit) is None: + pytest.skip( + "the registry is new in this pull request, which is merged with " + "a merge commit; its endpoint is admitted against an empty " + "registry above" + ) + assert registry.history_problems(ROOT, base_commit, head, full=False) == [] + + +def test_every_first_parent_commit_since_the_base_appends_only(): + if os.environ.get("GITHUB_EVENT_NAME") != "push": + pytest.skip( + "a pull request is checked at its merge commit; a branch's own " + "draft commits are not the base branch's history" + ) + assert registry.history_problems(ROOT, resolved_base()) == [] + + +def test_the_registry_stays_incomplete_until_the_inventory_lands(): + current = committed() + assert current["schema"] == registry.SCHEMA + if not (ROOT / current["completion_record"]).is_file(): + assert current["complete"] is False + assert any("incomplete" in rule for rule in current["rules"]) + + +def test_the_seeded_unit_keys_are_as_specified(): + entries = by_id(committed()) + for name, keys in EXPECTED_SEED_KEYS.items(): + assert entries[name]["unit_keys"] == keys + + +def test_the_seeded_disclosures_are_as_specified(): + pairs = { + tuple(sorted((entry["id"], other))) + for entry in committed()["entries"] + for other in entry["disclosed_overlaps"] + } + assert pairs == EXPECTED_DISCLOSURES + entries = by_id(committed()) + for a, b in EXPECTED_DISCLOSURES - {("H003", "H004"), ("H003", "H005")}: + assert registry.entries_overlap(entries[a], entries[b]) + + +def test_the_vocabulary_and_registrations_are_as_specified(): + assert { + source: declared["family"] + for source, declared in registry.SOURCES.items() + } == EXPECTED_FAMILIES + assert dict(registry.REGISTRATIONS_OUTSIDE_GATES_YAML) == ( + EXPECTED_OUTSIDE_GATES_YAML + ) + assert committed()["sources"] == { + source: dict(declared) for source, declared in registry.SOURCES.items() + } + + +def test_the_scored_runs_and_imports_are_as_specified(): + assert { + path: gate for path, (gate, _) in registry.SCORED_RUNS.items() + } == EXPECTED_SCORED_RUN_GATES + expected = { + name: { + "gate": gate, + "unit_keys": sorted(EXPECTED_SEED_KEYS[name]), + "scored_runs": sorted( + path + for path, owner in EXPECTED_SCORED_RUN_GATES.items() + if owner == gate + ), + } + for name, gate in EXPECTED_IMPORT_GATES.items() + } + assert { + name: { + "gate": authorized["gate"], + "unit_keys": sorted(authorized["unit_keys"]), + "scored_runs": sorted(authorized["scored_runs"]), + } + for name, authorized in registry.HISTORICAL_IMPORTS.items() + } == expected + for path, (_, digest) in registry.SCORED_RUNS.items(): + content = (ROOT / path).read_bytes() + assert hashlib.sha256(content).hexdigest() == digest, path + + +def test_every_used_entry_cites_committed_run_artifacts(): + for entry in committed()["entries"]: + if entry["status"] == "used": + assert entry["scored_evidence"], entry["id"] + for item in entry["scored_evidence"]: + relative = item["path"] + assert relative.startswith("runs/"), relative + assert relative.endswith(".json"), relative + assert (ROOT / relative).is_file(), relative diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 032190c9..ea9b7fb1 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 18125, - "artifact": 4261, + "unit": 18254, + "artifact": 4271, "integration_psid": 1450, "reproduction_legacy": 520, "oracle_policyengine": 308