From ca82fa4b25b9bc973c7e5055d301df700852ca7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 10:40:02 -0400 Subject: [PATCH 1/9] Track B: the held-out data registry, with its rules enforced on every commit The dynamics coordinator made one registry of held-out years, editions, tables and partitions the record for every Track B gate and every open-population gate (2026-10-21). This adds it at docs/design/track_b_holdout_registry.json with five verified prior uses, marked incomplete until B0.2's exposure inventory lands, and populace_dynamics.track_b.holdout_registry, which enforces its rules: schema v1, unique ids, symmetric disclosures, no silent shared unit, legal status histories, scored evidence for used entries, the incomplete flag, and append-only changes against the base branch. Tests: 25 unit tests, with Hypothesis properties; 16 of 16 mutants killed (docs/design/track_b_holdout_registry_mutations.json). Tiers: unit 9,958 -> 9,983. Co-Authored-By: Claude Opus 5.5 --- docs/design/track_b_holdout_registry.json | 229 ++++++++++ .../track_b_holdout_registry_mutations.json | 171 +++++++ .../track_b/holdout_registry.py | 334 ++++++++++++++ tests/README-tiers.md | 4 +- tests/test_track_b_holdout_registry.py | 424 ++++++++++++++++++ tests/tier_counts.json | 2 +- 6 files changed, 1161 insertions(+), 3 deletions(-) create mode 100644 docs/design/track_b_holdout_registry.json create mode 100644 docs/design/track_b_holdout_registry_mutations.json create mode 100644 src/populace_dynamics/track_b/holdout_registry.py create mode 100644 tests/test_track_b_holdout_registry.py diff --git a/docs/design/track_b_holdout_registry.json b/docs/design/track_b_holdout_registry.json new file mode 100644 index 000000000..98438c5e6 --- /dev/null +++ b/docs/design/track_b_holdout_registry.json @@ -0,0 +1,229 @@ +{ + "schema": "track_b_holdout_registry.v1", + "title": "Held-out data registry for Track B and the open-population lane", + "purpose": "The single registry of held-out source years, editions, tables and partitions for every Track B gate and every open-population gate, set up by the dynamics coordinator on 2026-10-21. It is part of Track B's B0.2 exposure record (design revision 3, section 5.2, elements 4 and 5). Cite it by path, commit and SHA-256. src/populace_dynamics/track_b/holdout_registry.py states and enforces its rules, and tests/test_track_b_holdout_registry.py runs them on every commit.", + "rules": [ + "A gate registers every source year, edition, table or partition it holds out here before its record freezes.", + "J's candidate holdouts are registered first, because J is on Track B's critical path. OP0.2 registers its picks for OP12's gate_ssa_totals here before it freezes them, and picks from the years J has not taken.", + "No unit may be held out by two gates silently. Each entry names its units as unit_keys, colon-separated from the source down; two keys share a unit when one equals the other or is a prefix of it. Live entries that share a unit list each other in disclosed_overlaps, and both gates' registrations name the overlap.", + "Reference years are recorded separately from publication or interview availability (design section 5.2, element 10).", + "An entry whose gate has scored has status 'used', and scored_evidence names the committed artifacts of that run. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", + "Entries are only appended. An entry's id, gate, lane, source, unit_kind, units and unit_keys never change; its history, evidence, scored_evidence and disclosed_overlaps only extend. A status change is a new history item with its date and reason. Ids are never reused.", + "The registry is incomplete until completion_record, B0.2's mechanical exposure inventory, exists. It starts with the prior uses verified below; the inventory adds the rest before any B0.2 gate record freezes." + ], + "status_values": { + "candidate": "proposed by a gate's B0.2 or OP0.2 record; not yet frozen", + "frozen": "fixed in a frozen gate record before that gate's candidates run", + "used": "scored by a registered gate; exposure for every later gate", + "withdrawn": "released before use, with the reason recorded" + }, + "complete": false, + "completion_record": "docs/design/track_b_b0_2_exposure_inventory.json", + "entries": [ + { + "id": "H001", + "gate": "gate_m6", + "lane": "M6 (locked gate)", + "status": "used", + "source": "PSID family and cross-year individual files", + "unit_kind": "reference_years", + "units": { + "boundary_T_star": 2014, + "held_out_window": "2015-2022", + "gated_earnings_reference_years": [ + 2016, + 2018 + ], + "gated_flow_event_years": [ + 2015, + 2016, + 2017, + 2018, + 2019 + ] + }, + "unit_keys": [ + "psid:reference_year:2015", + "psid:reference_year:2016", + "psid:reference_year:2017", + "psid:reference_year:2018", + "psid:reference_year:2019", + "psid:reference_year:2020", + "psid:reference_year:2021", + "psid:reference_year:2022" + ], + "evidence": [ + "gates.yaml:5374-5376 (fit <= T*=2014; score the held-out PSID window 2015-2022)", + "gates.yaml:5496-5501 (fit_holdout)" + ], + "scored_evidence": [ + "runs/gate_m6_candidate1_v1.json", + "runs/gate_m6_candidate2_v1.json", + "runs/gate_m6_candidate3_v1.json" + ], + "disclosed_overlaps": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32" + } + ] + }, + { + "id": "H002", + "gate": "B2", + "lane": "Track B", + "status": "used", + "source": "PSID family files FAM2013ER and FAM2015ER, with the cross-year individual file", + "unit_kind": "reference_years", + "units": { + "reference_years": [ + 2012, + 2014 + ], + "collection_waves": [ + 2013, + 2015 + ], + "ages": "25-64 at the collection wave", + "population": "2011 full-anchor persons with a valid 2010 row" + }, + "unit_keys": [ + "psid:reference_year:2012", + "psid:reference_year:2014" + ], + "evidence": [ + "src/populace_dynamics/track_b/b2.py:64-65", + "docs/design/track_b_b0_1_audit.md, section 16 (definitions)", + "registration: issue #42 comment 6252388925" + ], + "scored_evidence": [ + "runs/track_b_b2_score.json" + ], + "disclosed_overlaps": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "B2 scored on 2026-10-18; seeded at 1e739d32" + } + ] + }, + { + "id": "H003", + "gate": "gate_epuf_fill", + "lane": "EPUF career fill (locked gate)", + "status": "used", + "source": "SSA 2006 Earnings Public-Use File (EPUF)", + "unit_kind": "person_partition", + "units": { + "partition": "EPUF2006_TEST_by_salted_person_hash" + }, + "unit_keys": [ + "ssa_epuf_2006:partition:epuf2006_test_by_salted_person_hash" + ], + "evidence": [ + "gates.yaml:5918 (holdout_basis)" + ], + "scored_evidence": [ + "runs/epuf_fill_gate_test_v1.json" + ], + "disclosed_overlaps": [], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; open_population.md rev 2 plans gate_young_earn on a new salt, disjoint from this partition" + } + ] + }, + { + "id": "H004", + "gate": "gate_m4", + "lane": "M4 disability (locked gate)", + "status": "used", + "source": "PSID cross-year individual file employment status; SSA Annual Statistical Report on the SSDI Program, 2023", + "unit_kind": "publication_edition_tables", + "units": { + "holdout_basis": [ + "ind2023er_employment_status", + "di_asr_2023" + ] + }, + "unit_keys": [ + "psid:ind2023er:employment_status", + "ssa_di_asr:2023" + ], + "evidence": [ + "gates.yaml:2945 (holdout_basis)" + ], + "scored_evidence": [ + "runs/gate_m4_hazard_v1.json" + ], + "disclosed_overlaps": [ + "H005" + ], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; the edition's tables and the employment-status years are itemized by B0.2's inventory; shares di_asr_2023 with gate_w1 (H005)" + } + ] + }, + { + "id": "H005", + "gate": "gate_w1", + "lane": "W1 frame transport (locked gate)", + "status": "used", + "source": "SSA Annual Statistical Supplement 2023, section 6.B; SSA Annual Statistical Report on the SSDI Program, 2023; Census CPS tables; the certified 2024 frame", + "unit_kind": "publication_edition_tables", + "units": { + "holdout_basis": [ + "populace_us_2024_certified", + "ssa_supplement_6b", + "di_asr_2023", + "mermin_smith_committed_orderings" + ], + "tables_named": [ + "Supplement 2023 table 6.B5.1", + "Supplement 2023 tables 6.B4 and 6.B3", + "DI ASR 2023 table 19", + "Census HH-4, reference year 2023", + "Census AD-3, reference year 2023" + ] + }, + "unit_keys": [ + "ssa_supplement:2023:6.b5.1", + "ssa_supplement:2023:6.b4", + "ssa_supplement:2023:6.b3", + "ssa_di_asr:2023:table_19", + "census_cps:2023:hh-4", + "census_cps:2023:ad-3", + "populace_us_2024:certified", + "mermin_smith:committed_orderings" + ], + "evidence": [ + "gates.yaml:3735-3736 (holdout_basis)", + "gates.yaml:4582-4584 (sources)" + ], + "scored_evidence": [ + "runs/gate_w1_candidate1_v1.json", + "runs/gate_w1_candidate2_v1.json", + "runs/gate_w1_candidate3_v1.json", + "runs/gate_w1_candidate4_v1.json" + ], + "disclosed_overlaps": [ + "H004" + ], + "history": [ + { + "date": "2026-10-21", + "status": "used", + "note": "seeded from gates.yaml at 1e739d32; whether each named table was gated or report-only is itemized by B0.2's inventory; shares di_asr_2023 with gate_m4 (H004)" + } + ] + } + ] +} diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json new file mode 100644 index 000000000..661eeb88a --- /dev/null +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -0,0 +1,171 @@ +{ + "schema_version": 1, + "verification_class": "accounting conformance", + "admitted_scope": "The held-out data registry's rules on invented registries and on the committed registry; no population, behavioral or forecast claim.", + "mutation_method": "One deliberate production-line change per check; the registry test module is expected to fail; original bytes restored before the next check. No mutated implementation is committed.", + "restored_source_sha256": { + "src/populace_dynamics/track_b/holdout_registry.py": "6bfc2ce8beaccb837be050a51e73c051b767353399217667b0bc9ebf55bb1c63" + }, + "checks": [ + { + "mutation": "prefix overlap -> exact equality", + "old": "return left[:n] == right[:n]", + "new": "return left == right", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_the_shared_di_report_edition_is_disclosed_both_ways" + ] + }, + { + "mutation": "history dates may go back", + "old": "elif date < previous_date:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_history_dates_never_go_back" + ] + }, + { + "mutation": "any transition allowed", + "old": "status not in TRANSITIONS[previous_status]", + "new": "False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[statuses0-used-not" + ] + }, + { + "mutation": "status need not match history", + "old": "if entry[\"status\"] != previous_status:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[statuses3-candidate-is" + ] + }, + { + "mutation": "used needs no scored evidence", + "old": "if not entry[\"scored_evidence\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_used_entry_needs_committed_scored_evidence" + ] + }, + { + "mutation": "scored evidence need not exist", + "old": "if not _relative_file(repository, relative):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_used_entry_needs_committed_scored_evidence" + ] + }, + { + "mutation": "complete needs no record", + "old": "and registry[\"complete\"] is True", + "new": "and False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_completion_record" + ] + }, + { + "mutation": "repeated ids allowed", + "old": "if len(set(ids)) != len(ids):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_repeated_ids_are_refused" + ] + }, + { + "mutation": "one-way disclosure allowed", + "old": "elif entry[\"id\"] not in by_id[other][\"disclosed_overlaps\"]:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_one_way_disclosure_is_refused" + ] + }, + { + "mutation": "withdrawn entries counted live", + "old": "live = [entry for entry in entries if entry[\"status\"] != \"withdrawn\"]", + "new": "live = list(entries)", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_invented_registries_that_disclose_every_overlap_are_valid" + ] + }, + { + "mutation": "silent shared unit allowed", + "old": "other[\"id\"] not in entry[\"disclosed_overlaps\"]", + "new": "False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused" + ] + }, + { + "mutation": "complete may revert", + "old": "if base.get(\"complete\") is True and current.get(\"complete\") is not True:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_rules_and_completeness_only_move_forward" + ] + }, + { + "mutation": "entries may be removed or reordered", + "old": "if new_ids[: len(old_ids)] != old_ids:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused" + ] + }, + { + "mutation": "immutable fields may change", + "old": "if after.get(key) != before.get(key):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused" + ] + }, + { + "mutation": "extending lists may shrink", + "old": "if not _extends(before.get(key, []), after.get(key, [])):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_extending_lists_never_shrink" + ] + }, + { + "mutation": "absent base raises instead of None", + "old": "if listed.returncode == 0 and not listed.stdout.strip():", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_the_base_registry_is_read_at_the_merge_base" + ] + } + ] +} diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py new file mode 100644 index 000000000..19d20ae38 --- /dev/null +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -0,0 +1,334 @@ +"""The held-out data registry and the rules every commit must keep. + +``docs/design/track_b_holdout_registry.json`` records the source years, +editions, tables and partitions that Track B's gates and the +open-population lane's gates hold out. The dynamics coordinator made it +the single registry for every lane on 2026-10-21. This module enforces +its rules: + +- one schema, ``track_b_holdout_registry.v1``; +- unique entry ids, which are never removed, reordered or reused; +- no unit held out by two live entries silently: entries whose unit keys + overlap list each other in ``disclosed_overlaps``, and every disclosure + is symmetric; +- each entry's status is its history's last status, reached through legal + transitions only; +- a ``used`` entry names committed artifacts of the gate's scored run; +- entries only grow: against the base registry, an entry's identity and + units never change, and its history, evidence, scored evidence and + disclosures only extend; +- the registry stays marked incomplete until its completion record, B0.2's + mechanical exposure inventory, exists. + +Each check returns a list of problems; an empty list means the registry +keeps the rules. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +SCHEMA = "track_b_holdout_registry.v1" +REGISTRY = "docs/design/track_b_holdout_registry.json" +STATUSES = ("candidate", "frozen", "used", "withdrawn") +#: Legal next statuses. A first history item may hold any status, so prior +#: uses are recorded as ``used`` when they are entered. +TRANSITIONS = { + "candidate": frozenset({"frozen", "withdrawn"}), + "frozen": frozenset({"used", "withdrawn"}), + "used": frozenset(), + "withdrawn": frozenset(), +} +UNIT_KINDS = ( + "reference_years", + "publication_edition_tables", + "person_partition", +) +TOP_LEVEL = ( + "schema", + "title", + "purpose", + "rules", + "status_values", + "complete", + "completion_record", + "entries", +) +ENTRY_KEYS = ( + "id", + "gate", + "lane", + "status", + "source", + "unit_kind", + "units", + "unit_keys", + "evidence", + "scored_evidence", + "disclosed_overlaps", + "history", +) +#: Fields that identify an entry and its units; they never change. +IMMUTABLE = ("id", "gate", "lane", "source", "unit_kind", "units", "unit_keys") +#: Lists that may only extend. +EXTENDING = ("evidence", "scored_evidence", "disclosed_overlaps", "history") +ID_RX = re.compile(r"H[0-9]{3,}") +SEGMENT = r"[a-z0-9][a-z0-9_.\-]*" +UNIT_KEY_RX = re.compile(rf"{SEGMENT}(?::{SEGMENT})*") +DATE_RX = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}") + + +def load(repository: Path) -> dict[str, Any]: + """The registry in ``repository``'s working tree.""" + return json.loads((repository / REGISTRY).read_text(encoding="utf-8")) + + +def keys_overlap(a: str, b: str) -> bool: + """Whether two unit keys name a shared unit: one is the other or a + ``:``-segment prefix of it, so ``ssa_di_asr:2023`` (an edition) + overlaps ``ssa_di_asr:2023:table_19`` (one of its tables).""" + left, right = a.split(":"), b.split(":") + n = min(len(left), len(right)) + return left[:n] == right[:n] + + +def entries_overlap(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: + """Whether two entries hold out a shared unit.""" + return any( + keys_overlap(x, y) for x in a["unit_keys"] for y in b["unit_keys"] + ) + + +def _relative_file(repository: Path, relative: Any) -> bool: + """``relative`` names a file inside ``repository``.""" + if not isinstance(relative, str) or not relative: + return False + if relative.startswith(("/", "~")) or ".." in relative.split("/"): + return False + return (repository / relative).is_file() + + +def _entry_problems( + entry: Mapping[str, Any], repository: Path | None +) -> list[str]: + """Problems inside one entry, without reference to the others.""" + name = entry.get("id", "") + missing = [key for key in ENTRY_KEYS if key not in entry] + if missing: + return [f"{name}: missing {', '.join(missing)}"] + problems: list[str] = [] + if not isinstance(name, str) or not ID_RX.fullmatch(name): + problems.append(f"{name}: id must match H followed by 3+ digits") + for key in ("gate", "lane", "source"): + if not isinstance(entry[key], str) or not entry[key]: + problems.append(f"{name}: {key} must be a non-empty string") + if entry["unit_kind"] not in UNIT_KINDS: + problems.append(f"{name}: unknown unit_kind {entry['unit_kind']!r}") + if not isinstance(entry["units"], Mapping) or not entry["units"]: + problems.append(f"{name}: units must be a non-empty mapping") + keys = entry["unit_keys"] + if ( + not isinstance(keys, list) + or not keys + or not all( + isinstance(k, str) and UNIT_KEY_RX.fullmatch(k) for k in keys + ) + ): + problems.append(f"{name}: unit_keys must be well-formed keys") + elif len(set(keys)) != len(keys): + problems.append(f"{name}: unit_keys repeat") + for key in ("evidence", "scored_evidence", "disclosed_overlaps"): + if not isinstance(entry[key], list) or not all( + isinstance(item, str) and item for item in entry[key] + ): + problems.append(f"{name}: {key} must be a list of strings") + history = entry["history"] + if not isinstance(history, list) or not history: + problems.append(f"{name}: history must be a non-empty list") + return problems + previous_status, previous_date = None, "" + for index, item in enumerate(history): + if not isinstance(item, Mapping): + problems.append(f"{name}: history[{index}] is not a mapping") + return problems + status, date = item.get("status"), item.get("date") + if status not in STATUSES: + problems.append(f"{name}: history[{index}] status {status!r}") + return problems + if not isinstance(date, str) or not DATE_RX.fullmatch(date): + problems.append(f"{name}: history[{index}] date {date!r}") + elif date < previous_date: + problems.append(f"{name}: history[{index}] goes back in time") + else: + previous_date = date + if not isinstance(item.get("note"), str) or not item["note"]: + problems.append(f"{name}: history[{index}] needs a note") + if previous_status is not None and ( + status not in TRANSITIONS[previous_status] + ): + problems.append( + f"{name}: history[{index}] moves {previous_status} -> " + f"{status}, which is not a legal transition" + ) + previous_status = status + if entry["status"] != previous_status: + problems.append( + f"{name}: status {entry['status']!r} is not its history's " + f"last status {previous_status!r}" + ) + if entry["status"] == "used": + if not entry["scored_evidence"]: + problems.append(f"{name}: a used entry needs scored_evidence") + elif repository is not None: + for relative in entry["scored_evidence"]: + if not _relative_file(repository, relative): + problems.append( + f"{name}: scored_evidence {relative!r} is not a " + "committed file" + ) + return problems + + +def validate( + registry: Mapping[str, Any], repository: Path | None = None +) -> list[str]: + """Every rule that one registry must keep on its own. With + ``repository``, scored evidence and the completion record are checked + against its files.""" + missing = [key for key in TOP_LEVEL if key not in registry] + if missing: + return [f"registry: missing {', '.join(missing)}"] + problems: list[str] = [] + if registry["schema"] != SCHEMA: + problems.append(f"registry: schema {registry['schema']!r}") + if not isinstance(registry["rules"], list) or not all( + isinstance(rule, str) and rule for rule in registry["rules"] + ): + problems.append("registry: rules must be a list of strings") + if not isinstance(registry["status_values"], Mapping) or set( + registry["status_values"] + ) != set(STATUSES): + problems.append("registry: status_values must name each status") + if not isinstance(registry["complete"], bool): + problems.append("registry: complete must be true or false") + record = registry["completion_record"] + if not isinstance(record, str) or not record: + problems.append("registry: completion_record must name a path") + elif ( + repository is not None + and registry["complete"] is True + and not _relative_file(repository, record) + ): + problems.append( + f"registry: marked complete, but {record} does not exist" + ) + entries = registry["entries"] + if not isinstance(entries, list): + return [*problems, "registry: entries must be a list"] + for entry in entries: + if not isinstance(entry, Mapping): + return [*problems, "registry: an entry is not a mapping"] + problems.extend(_entry_problems(entry, repository)) + if problems: + return problems + ids = [entry["id"] for entry in entries] + if len(set(ids)) != len(ids): + problems.append("registry: entry ids repeat") + return problems + by_id = {entry["id"]: entry for entry in entries} + for entry in entries: + for other in entry["disclosed_overlaps"]: + if other == entry["id"]: + problems.append(f"{entry['id']}: discloses itself") + elif other not in by_id: + problems.append(f"{entry['id']}: discloses unknown {other}") + elif entry["id"] not in by_id[other]["disclosed_overlaps"]: + problems.append( + f"{entry['id']}: discloses {other}, which does not " + "disclose it back" + ) + live = [entry for entry in entries if entry["status"] != "withdrawn"] + for index, entry in enumerate(live): + for other in live[index + 1 :]: + if entries_overlap(entry, other) and ( + other["id"] not in entry["disclosed_overlaps"] + ): + problems.append( + f"{entry['id']} and {other['id']} hold out a shared " + "unit without disclosing it" + ) + return problems + + +def _extends(base: Sequence[Any], current: Sequence[Any]) -> bool: + """``current`` keeps ``base`` as its prefix.""" + return list(current[: len(base)]) == list(base) + + +def append_only_problems( + base: Mapping[str, Any], current: Mapping[str, Any] +) -> list[str]: + """How ``current`` breaks the append-only rule against ``base``, an + earlier registry that kept the rules.""" + problems: list[str] = [] + for key in ("schema", "status_values", "completion_record"): + if current.get(key) != base.get(key): + problems.append(f"registry: {key} changed") + if not _extends(base.get("rules", []), current.get("rules", [])): + problems.append("registry: rules may only be appended") + if base.get("complete") is True and current.get("complete") is not True: + problems.append("registry: a complete registry became incomplete") + old = base.get("entries", []) + new = current.get("entries", []) + old_ids = [entry.get("id") for entry in old] + new_ids = [entry.get("id") for entry in new] + if new_ids[: len(old_ids)] != old_ids: + problems.append( + "registry: earlier entries were removed, reordered or replaced" + ) + return problems + for before, after in zip(old, new[: len(old)], strict=True): + for key in IMMUTABLE: + if after.get(key) != before.get(key): + problems.append(f"{before['id']}: {key} changed") + for key in EXTENDING: + if not _extends(before.get(key, []), after.get(key, [])): + problems.append(f"{before['id']}: {key} may only extend") + return problems + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", str(repository), *args], + capture_output=True, + text=True, + check=False, + ) + + +def base_registry(repository: Path) -> dict[str, Any] | None: + """The registry at the merge base of ``HEAD`` and the base branch + (``GITHUB_BASE_REF`` on a pull request, otherwise ``master``), or + ``None`` when the base has no registry yet. Raises ``LookupError`` + when no base can be resolved.""" + branch = os.environ.get("GITHUB_BASE_REF") or "master" + for ref in (f"origin/{branch}", branch): + merge_base = _git(repository, "merge-base", "HEAD", ref) + if merge_base.returncode == 0: + break + else: + raise LookupError(f"no merge base with {branch} in {repository}") + commit = merge_base.stdout.strip() + shown = _git(repository, "show", f"{commit}:{REGISTRY}") + if shown.returncode != 0: + listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) + if listed.returncode == 0 and not listed.stdout.strip(): + return None + raise LookupError(f"cannot read {REGISTRY} at {commit}") + return json.loads(shown.stdout) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 8b52da848..af067eacc 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,9 +39,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 9,958 | +| `unit` | 9,983 | | `artifact` | 4,253 | | `integration_psid` | 1,450 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 308 | -| **Total** | **16,489** | +| **Total** | **16,514** | diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py new file mode 100644 index 000000000..5ee2e0432 --- /dev/null +++ b/tests/test_track_b_holdout_registry.py @@ -0,0 +1,424 @@ +"""The held-out data registry keeps its rules on every commit. + +``docs/design/track_b_holdout_registry.json`` is the single registry of +held-out years, editions, tables and partitions for Track B's gates and +the open-population lane's gates (dynamics coordinator, 2026-10-21). +``populace_dynamics.track_b.holdout_registry`` states the rules; these +tests run them on the committed registry, against its base branch, and +as properties over invented registries. +""" + +from __future__ import annotations + +import copy +import json +import os +import subprocess +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from populace_dynamics.track_b import holdout_registry as registry + +ROOT = Path(__file__).resolve().parents[1] + + +def committed() -> dict: + return registry.load(ROOT) + + +def test_the_committed_registry_keeps_its_rules(): + assert registry.validate(committed(), ROOT) == [] + + +def test_the_committed_registry_only_appends_to_its_base(): + try: + base = registry.base_registry(ROOT) + except LookupError as error: + if os.environ.get("GITHUB_ACTIONS") == "true": + pytest.fail(f"CI must resolve the base registry: {error}") + pytest.skip(f"no base branch here: {error}") + if base is None: + return + assert registry.validate(base) == [] + assert registry.append_only_problems(base, committed()) == [] + + +def test_the_registry_stays_incomplete_until_the_inventory_lands(): + current = committed() + assert current["schema"] == registry.SCHEMA + record = ROOT / current["completion_record"] + if not record.is_file(): + assert current["complete"] is False + assert any("incomplete" in rule for rule in current["rules"]) + + +def test_the_shared_di_report_edition_is_disclosed_both_ways(): + by_id = {entry["id"]: entry for entry in committed()["entries"]} + m4, w1 = by_id["H004"], by_id["H005"] + assert registry.entries_overlap(m4, w1) + assert "H005" in m4["disclosed_overlaps"] + assert "H004" in w1["disclosed_overlaps"] + + +def test_every_used_entry_names_committed_scored_artifacts(): + for entry in committed()["entries"]: + if entry["status"] == "used": + assert entry["scored_evidence"], entry["id"] + for relative in entry["scored_evidence"]: + assert (ROOT / relative).is_file(), relative + + +# Invented registries ------------------------------------------------------ + +KEY_POOL = ( + "alpha", + "alpha:2020", + "alpha:2021", + "alpha:2021:t1", + "beta:x", + "beta:y", + "beta:y:z", + "gamma:p", +) +PATHS = { + "candidate": ["candidate"], + "frozen": ["candidate", "frozen"], + "used": ["candidate", "frozen", "used"], + "withdrawn": ["candidate", "withdrawn"], + "seeded_used": ["used"], +} + + +def make_entry(index: int, keys: list[str], path: str) -> dict: + statuses = PATHS[path] + return { + "id": f"H{index + 1:03d}", + "gate": f"gate_{index}", + "lane": "invented", + "status": statuses[-1], + "source": "invented source", + "unit_kind": "reference_years", + "units": {"years": [index]}, + "unit_keys": keys, + "evidence": [f"evidence {index}"], + "scored_evidence": ( + [f"scores/score_{index}.txt"] if statuses[-1] == "used" else [] + ), + "disclosed_overlaps": [], + "history": [ + {"date": f"2026-10-{20 + step:02d}", "status": s, "note": "n"} + for step, s in enumerate(statuses) + ], + } + + +def disclose_all(entries: list[dict]) -> None: + """Disclose every overlap between live entries, both ways.""" + live = [entry for entry in entries if entry["status"] != "withdrawn"] + for index, entry in enumerate(live): + for other in live[index + 1 :]: + if registry.entries_overlap(entry, other): + if other["id"] not in entry["disclosed_overlaps"]: + entry["disclosed_overlaps"].append(other["id"]) + if entry["id"] not in other["disclosed_overlaps"]: + other["disclosed_overlaps"].append(entry["id"]) + + +def make_registry(entries: list[dict]) -> dict: + return { + "schema": registry.SCHEMA, + "title": "invented", + "purpose": "invented", + "rules": ["rule one", "the registry is incomplete"], + "status_values": {status: status for status in registry.STATUSES}, + "complete": False, + "completion_record": "docs/design/inventory.json", + "entries": entries, + } + + +entry_specs = st.lists( + st.tuples( + st.lists( + st.sampled_from(KEY_POOL), min_size=1, max_size=3, unique=True + ), + st.sampled_from(sorted(PATHS)), + ), + min_size=0, + max_size=6, +) + + +def build(specs) -> dict: + entries = [ + make_entry(i, keys, path) for i, (keys, path) in enumerate(specs) + ] + disclose_all(entries) + return make_registry(entries) + + +@settings(max_examples=150, deadline=None) +@given(entry_specs) +def test_invented_registries_that_disclose_every_overlap_are_valid(specs): + assert registry.validate(build(specs)) == [] + + +@settings(max_examples=150, deadline=None) +@given(entry_specs, st.data()) +def test_a_one_way_disclosure_is_refused(specs, data): + current = build(specs) + disclosing = [e for e in current["entries"] if e["disclosed_overlaps"]] + if not disclosing: + return + entry = data.draw(st.sampled_from(disclosing)) + entry["disclosed_overlaps"].pop() + problems = registry.validate(current) + assert problems + assert any( + "does not disclose it back" in p or "without disclosing" in p + for p in problems + ) + + +@settings(max_examples=150, deadline=None) +@given(entry_specs) +def test_a_silent_shared_unit_is_refused(specs): + current = build(specs) + for entry in current["entries"]: + entry["disclosed_overlaps"] = [] + live = [e for e in current["entries"] if e["status"] != "withdrawn"] + shared = any( + registry.entries_overlap(a, b) + for i, a in enumerate(live) + for b in live[i + 1 :] + ) + problems = registry.validate(current) + assert bool(problems) == shared + if shared: + assert any("without disclosing" in p for p in problems) + + +@settings(max_examples=150, deadline=None) +@given(entry_specs, entry_specs) +def test_appending_disclosed_entries_keeps_every_rule(base_specs, more): + base = build(base_specs) + current = copy.deepcopy(base) + start = len(current["entries"]) + for offset, (keys, path) in enumerate(more): + current["entries"].append(make_entry(start + offset, keys, path)) + disclose_all(current["entries"]) + assert registry.validate(current) == [] + assert registry.append_only_problems(base, current) == [] + + +IMMUTABLE_EDITS = { + "id": "H999", + "gate": "gate_other", + "lane": "other lane", + "source": "other source", + "unit_kind": "person_partition", + "units": {"years": [-1]}, + "unit_keys": ["delta:other"], +} + + +@settings(max_examples=150, deadline=None) +@given( + entry_specs.filter(bool), + st.sampled_from(registry.IMMUTABLE), + st.data(), +) +def test_changing_an_earlier_entry_is_refused(specs, key, data): + base = build(specs) + current = copy.deepcopy(base) + entry = data.draw(st.sampled_from(current["entries"])) + entry[key] = IMMUTABLE_EDITS[key] + assert registry.append_only_problems(base, current) + + +@settings(max_examples=150, deadline=None) +@given(entry_specs.filter(bool), st.data()) +def test_removing_or_reordering_entries_is_refused(specs, data): + base = build(specs) + removed = copy.deepcopy(base) + del removed["entries"][data.draw(st.integers(0, len(specs) - 1))] + assert registry.append_only_problems(base, removed) + if len(specs) > 1: + reordered = copy.deepcopy(base) + reordered["entries"].reverse() + assert registry.append_only_problems(base, reordered) + + +@settings(max_examples=150, deadline=None) +@given( + entry_specs.filter(bool), + st.sampled_from(registry.EXTENDING), + st.data(), +) +def test_extending_lists_never_shrink(specs, key, data): + base = build(specs) + current = copy.deepcopy(base) + entry = data.draw(st.sampled_from(current["entries"])) + if not entry[key]: + entry[key] = ["something new"] + base_entry = base["entries"][current["entries"].index(entry)] + base_entry[key] = ["something old"] + else: + entry[key] = entry[key][1:] + assert registry.append_only_problems(base, current) + + +def test_rules_and_completeness_only_move_forward(): + base = make_registry([]) + trimmed = copy.deepcopy(base) + trimmed["rules"] = trimmed["rules"][1:] + assert registry.append_only_problems(base, trimmed) + done = copy.deepcopy(base) + done["complete"] = True + assert registry.append_only_problems(base, done) == [] + assert registry.append_only_problems(done, base) + renamed = copy.deepcopy(base) + renamed["schema"] = "track_b_holdout_registry.v2" + assert registry.append_only_problems(base, renamed) + + +def test_repeated_ids_are_refused(): + current = build([(["alpha"], "candidate"), (["beta:x"], "candidate")]) + current["entries"][1]["id"] = "H001" + assert any("ids repeat" in p for p in registry.validate(current)) + + +@pytest.mark.parametrize( + "statuses, last, message", + [ + (["candidate", "used"], "used", "not a legal transition"), + (["used", "candidate"], "candidate", "not a legal transition"), + (["withdrawn", "frozen"], "frozen", "not a legal transition"), + (["candidate", "frozen"], "candidate", "is not its history's last"), + ], +) +def test_status_follows_a_legal_history(statuses, last, message): + entry = make_entry(0, ["alpha"], "candidate") + entry["history"] = [ + {"date": "2026-10-21", "status": s, "note": "n"} for s in statuses + ] + entry["status"] = last + entry["scored_evidence"] = ["x"] if last == "used" else [] + assert any(message in p for p in registry.validate(make_registry([entry]))) + + +def test_history_dates_never_go_back(): + entry = make_entry(0, ["alpha"], "frozen") + entry["history"][1]["date"] = "2026-10-01" + problems = registry.validate(make_registry([entry])) + assert any("goes back in time" in p for p in problems) + + +def test_a_used_entry_needs_committed_scored_evidence(tmp_path): + entry = make_entry(0, ["alpha"], "used") + current = make_registry([entry]) + assert registry.validate(current) == [] + assert any( + "not a committed file" in p + for p in registry.validate(current, tmp_path) + ) + score = tmp_path / entry["scored_evidence"][0] + score.parent.mkdir(parents=True) + score.write_text("scored", encoding="utf-8") + assert registry.validate(current, tmp_path) == [] + entry["scored_evidence"] = [] + assert any( + "needs scored_evidence" in p + for p in registry.validate(current, tmp_path) + ) + entry["scored_evidence"] = ["../outside.txt"] + assert any( + "not a committed file" in p + for p in registry.validate(current, tmp_path) + ) + + +def test_complete_needs_its_completion_record(tmp_path): + current = make_registry([]) + current["complete"] = True + assert any( + "does not exist" in p for p in registry.validate(current, tmp_path) + ) + record = tmp_path / current["completion_record"] + record.parent.mkdir(parents=True) + record.write_text("{}", encoding="utf-8") + assert registry.validate(current, tmp_path) == [] + + +@given(st.sampled_from(KEY_POOL), st.sampled_from(KEY_POOL)) +def test_key_overlap_is_symmetric_and_prefix_based(a, b): + assert registry.keys_overlap(a, a) + assert registry.keys_overlap(a, b) == registry.keys_overlap(b, a) + left, right = a.split(":"), b.split(":") + shorter = min(len(left), len(right)) + assert registry.keys_overlap(a, b) == (left[:shorter] == right[:shorter]) + + +def test_an_edition_overlaps_its_tables_but_not_its_neighbours(): + assert registry.keys_overlap("ssa_di_asr:2023", "ssa_di_asr:2023:table_19") + assert not registry.keys_overlap("ssa_di_asr:2023", "ssa_di_asr:2024") + assert not registry.keys_overlap("alpha:2021", "alpha:2021x") + + +# The base registry, read through git -------------------------------------- + + +def git(repository: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *args], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +def init_repository(tmp_path: Path) -> Path: + repository = tmp_path / "repo" + repository.mkdir() + git(repository, "init", "-q", "-b", "master") + git(repository, "config", "user.email", "t@example.com") + git(repository, "config", "user.name", "t") + (repository / "README").write_text("r", encoding="utf-8") + git(repository, "add", "README") + git(repository, "commit", "-q", "-m", "start") + return repository + + +def write_registry(repository: Path, current: dict) -> None: + path = repository / registry.REGISTRY + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(current), encoding="utf-8") + + +def test_the_base_registry_is_read_at_the_merge_base(tmp_path, monkeypatch): + monkeypatch.delenv("GITHUB_BASE_REF", raising=False) + repository = init_repository(tmp_path) + git(repository, "checkout", "-q", "-b", "feature") + assert registry.base_registry(repository) is None + git(repository, "checkout", "-q", "master") + first = build([(["alpha"], "candidate")]) + write_registry(repository, first) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "registry") + git(repository, "checkout", "-q", "-b", "feature-2") + second = copy.deepcopy(first) + second["entries"].append(make_entry(1, ["beta:x"], "candidate")) + write_registry(repository, second) + git(repository, "commit", "-q", "-am", "append") + assert registry.base_registry(repository) == first + + +def test_no_base_branch_raises_rather_than_passing(tmp_path, monkeypatch): + monkeypatch.setenv("GITHUB_BASE_REF", "no-such-branch") + repository = init_repository(tmp_path) + with pytest.raises(LookupError): + registry.base_registry(repository) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 3841240d4..b41580fd1 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 9958, + "unit": 9983, "artifact": 4253, "integration_psid": 1450, "reproduction_legacy": 520, From c299a30443b5704f56606cd43387960c51a2f14f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 21:45:10 -0400 Subject: [PATCH 2/9] Holdout registry: answer review round 1 (one high, five medium, one low) - High: gate_m6's entry kept event years and reference years apart only in prose. Unit keys now name a declared source and its years; PSID is keyed by interview wave, and each entry's units state the gated and report-only reference and event years separately. - Overlap: two keys share a unit when the source matches and the years intersect; scope segments never narrow it, so table_19 and table19 overlap. Sources are a declared vocabulary whose catalog references must differ, which refuses alias names. Same-family relations the rule cannot see need an overlap_review entry with a reason. The coarser keys expose three more real shared units (gate_m6 with B2 and gate_m4, B2 with gate_m4), now disclosed both ways. - Used means scored: scored_evidence is structured (path, field, equals); the file must be committed, inside the repository and not a symlink, and its field must equal one of the entry's gate identifiers, each of which belongs to one gate only. - Master pushes compare against the push's before commit, or the first parent when the merge base is HEAD itself. - Registrations start as a lone candidate in a commit of their own; historical imports start as frozen or used and close when the registry is complete. - The immutable and extending field lists are specified independently in the tests; every field is typed and dates are parsed, so malformed input is reported rather than raised (fuzzed). - The birth-evidence seal excludes the new module, as it does b2.py. Tests: 62 unit tests; 36 of 36 mutants killed. Tiers: unit 9,958 -> 10,020. Co-Authored-By: Claude Opus 5.5 --- docs/design/track_b_holdout_registry.json | 268 ++++++-- .../track_b_holdout_registry_mutations.json | 313 +++++++-- scripts/first_estimates_birth_evidence.py | 4 + .../track_b/holdout_registry.py | 585 ++++++++++++---- tests/README-tiers.md | 4 +- .../estimates/test_birth_evidence_artifact.py | 2 + tests/test_track_b_holdout_registry.py | 644 +++++++++++++----- tests/tier_counts.json | 2 +- 8 files changed, 1417 insertions(+), 405 deletions(-) diff --git a/docs/design/track_b_holdout_registry.json b/docs/design/track_b_holdout_registry.json index 98438c5e6..03cd35555 100644 --- a/docs/design/track_b_holdout_registry.json +++ b/docs/design/track_b_holdout_registry.json @@ -1,97 +1,172 @@ { "schema": "track_b_holdout_registry.v1", "title": "Held-out data registry for Track B and the open-population lane", - "purpose": "The single registry of held-out source years, editions, tables and partitions for every Track B gate and every open-population gate, set up by the dynamics coordinator on 2026-10-21. It is part of Track B's B0.2 exposure record (design revision 3, section 5.2, elements 4 and 5). Cite it by path, commit and SHA-256. src/populace_dynamics/track_b/holdout_registry.py states and enforces its rules, and tests/test_track_b_holdout_registry.py runs them on every commit.", + "purpose": "The single registry of the data that every Track B gate and every open-population gate holds out, set up by the dynamics coordinator on 2026-10-21. It is part of Track B's B0.2 exposure record (design revision 3, section 5.2, elements 4 and 5). Cite it by path, commit and SHA-256. src/populace_dynamics/track_b/holdout_registry.py states and enforces its rules, and tests/test_track_b_holdout_registry.py runs them on every commit.", "rules": [ - "A gate registers every source year, edition, table or partition it holds out here before its record freezes.", + "A gate registers what it holds out here, as a lone candidate entry in a commit of its own, before its record freezes. It moves to frozen, used or withdrawn only in later commits.", "J's candidate holdouts are registered first, because J is on Track B's critical path. OP0.2 registers its picks for OP12's gate_ssa_totals here before it freezes them, and picks from the years J has not taken.", - "No unit may be held out by two gates silently. Each entry names its units as unit_keys, colon-separated from the source down; two keys share a unit when one equals the other or is a prefix of it. Live entries that share a unit list each other in disclosed_overlaps, and both gates' registrations name the overlap.", - "Reference years are recorded separately from publication or interview availability (design section 5.2, element 10).", - "An entry whose gate has scored has status 'used', and scored_evidence names the committed artifacts of that run. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", - "Entries are only appended. An entry's id, gate, lane, source, unit_kind, units and unit_keys never change; its history, evidence, scored_evidence and disclosed_overlaps only extend. A status change is a new history item with its date and reason. Ids are never reused.", + "Every unit key names a declared source, then its years (a year, an inclusive range, or 'undated'), then optional scope segments. Each source declares a family, a title, a catalog reference no other source shares, and what its key year means. A new source is added only if it is not another name for a declared one.", + "Two unit keys share a unit when their source is the same and their years intersect; scope segments describe the unit and never narrow the overlap. Live entries that share a unit list each other in disclosed_overlaps, and both gates' registrations name the overlap. No unit is held out by two gates silently.", + "The overlap rule cannot see two sources that share underlying data, or two editions of a publication that report the same data years. So an entry lists in overlap_review, with a reason, every earlier live entry of the same source family that it neither shares a unit with nor discloses.", + "Reference years are recorded separately from interview or publication availability (design section 5.2, element 10): a unit key's year is what its source declares (for PSID, the interview wave), and each entry's units state the reference or event years.", + "An entry whose gate has scored has status 'used'. Its scored_evidence cites committed artifacts of that run, each with a field whose value identifies the gate. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", + "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused.", + "A historical import records a gate that froze or scored before this registry existed. Imports are allowed only while the registry is incomplete.", "The registry is incomplete until completion_record, B0.2's mechanical exposure inventory, exists. It starts with the prior uses verified below; the inventory adds the rest before any B0.2 gate record freezes." ], "status_values": { - "candidate": "proposed by a gate's B0.2 or OP0.2 record; not yet frozen", + "candidate": "registered by a gate's B0.2 or OP0.2 record; not yet frozen", "frozen": "fixed in a frozen gate record before that gate's candidates run", "used": "scored by a registered gate; exposure for every later gate", "withdrawn": "released before use, with the reason recorded" }, + "sources": { + "psid": { + "family": "psid", + "title": "Panel Study of Income Dynamics: family files, cross-year individual file and supplements", + "catalog": "University of Michigan Institute for Social Research: Panel Study of Income Dynamics", + "key_year": "interview wave (collection year); income reference years are the wave year minus one" + }, + "ssa_epuf": { + "family": "ssa_administrative", + "title": "SSA Earnings Public-Use File", + "catalog": "SSA Office of Research, Evaluation, and Statistics: Earnings Public-Use File", + "key_year": "file release year" + }, + "ssa_supplement": { + "family": "ssa_administrative", + "title": "Annual Statistical Supplement to the Social Security Bulletin", + "catalog": "SSA Office of Research, Evaluation, and Statistics: Annual Statistical Supplement to the Social Security Bulletin", + "key_year": "edition year" + }, + "ssa_di_asr": { + "family": "ssa_administrative", + "title": "Annual Statistical Report on the Social Security Disability Insurance Program", + "catalog": "SSA Office of Research, Evaluation, and Statistics: Annual Statistical Report on the Social Security Disability Insurance Program", + "key_year": "edition year" + }, + "census_cps_tables": { + "family": "us_cross_section", + "title": "U.S. Census Bureau Current Population Survey published tables", + "catalog": "U.S. Census Bureau: Current Population Survey published tables", + "key_year": "table reference year" + }, + "populace_us_frame": { + "family": "us_cross_section", + "title": "Microcosm US certified frame", + "catalog": "hf://policyengine/populace-us/populace_us_2024.h5", + "key_year": "frame vintage" + }, + "mermin_smith_anchors": { + "family": "literature", + "title": "Mermin and Smith published orderings, committed as anchors", + "catalog": "gates.yaml gate_w1 holdout_basis: mermin_smith_committed_orderings", + "key_year": "undated" + } + }, "complete": false, "completion_record": "docs/design/track_b_b0_2_exposure_inventory.json", "entries": [ { "id": "H001", + "provenance": "historical_import", "gate": "gate_m6", + "gate_identifiers": [ + "gate_m6", + "m6_temporal_holdout_projection_drift" + ], "lane": "M6 (locked gate)", "status": "used", "source": "PSID family and cross-year individual files", - "unit_kind": "reference_years", "units": { "boundary_T_star": 2014, + "seed_wave": 2015, "held_out_window": "2015-2022", - "gated_earnings_reference_years": [ - 2016, - 2018 - ], "gated_flow_event_years": [ 2015, 2016, 2017, 2018, 2019 - ] + ], + "gated_earnings_reference_years": [ + 2016, + 2018 + ], + "report_only_shock_earnings_reference_years": [ + 2020, + 2022 + ], + "report_only_shock_flow_event_years": [ + 2020, + 2021 + ], + "interview_waves": "2015 (seed) to 2023" }, "unit_keys": [ - "psid:reference_year:2015", - "psid:reference_year:2016", - "psid:reference_year:2017", - "psid:reference_year:2018", - "psid:reference_year:2019", - "psid:reference_year:2020", - "psid:reference_year:2021", - "psid:reference_year:2022" + "psid:2015-2023:held_out_window" ], "evidence": [ "gates.yaml:5374-5376 (fit <= T*=2014; score the held-out PSID window 2015-2022)", - "gates.yaml:5496-5501 (fit_holdout)" + "gates.yaml:5496-5508 (fit_holdout and shock_window)" ], "scored_evidence": [ - "runs/gate_m6_candidate1_v1.json", - "runs/gate_m6_candidate2_v1.json", - "runs/gate_m6_candidate3_v1.json" + { + "path": "runs/gate_m6_candidate1_v1.json", + "field": "gate.name", + "equals": "gate_m6" + }, + { + "path": "runs/gate_m6_candidate2_v1.json", + "field": "gate.name", + "equals": "gate_m6" + }, + { + "path": "runs/gate_m6_candidate3_v1.json", + "field": "gate.name", + "equals": "gate_m6" + } ], - "disclosed_overlaps": [], + "disclosed_overlaps": [ + "H002", + "H004" + ], + "overlap_review": [], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32" + "note": "seeded from gates.yaml at 1e739d32; shares PSID interview wave 2015 with B2 (H002) and waves 2015-2023 with gate_m4's panel (H004)" } ] }, { "id": "H002", + "provenance": "historical_import", "gate": "B2", + "gate_identifiers": [ + "B2", + "6252388925" + ], "lane": "Track B", "status": "used", "source": "PSID family files FAM2013ER and FAM2015ER, with the cross-year individual file", - "unit_kind": "reference_years", "units": { "reference_years": [ 2012, 2014 ], - "collection_waves": [ + "interview_waves": [ 2013, 2015 ], + "anchor_wave_not_held_out": 2011, "ages": "25-64 at the collection wave", "population": "2011 full-anchor persons with a valid 2010 row" }, "unit_keys": [ - "psid:reference_year:2012", - "psid:reference_year:2014" + "psid:2013:fam2013er", + "psid:2015:fam2015er" ], "evidence": [ "src/populace_dynamics/track_b/b2.py:64-65", @@ -99,86 +174,136 @@ "registration: issue #42 comment 6252388925" ], "scored_evidence": [ - "runs/track_b_b2_score.json" + { + "path": "runs/track_b_b2_score.json", + "field": "registration_id", + "equals": "6252388925" + } ], - "disclosed_overlaps": [], + "disclosed_overlaps": [ + "H001", + "H004" + ], + "overlap_review": [], "history": [ { "date": "2026-10-21", "status": "used", - "note": "B2 scored on 2026-10-18; seeded at 1e739d32" + "note": "B2 scored on 2026-10-18; seeded at 1e739d32; its 2015 wave is gate_m6's seed wave (H001), and both waves fall in gate_m4's panel (H004)" } ] }, { "id": "H003", + "provenance": "historical_import", "gate": "gate_epuf_fill", + "gate_identifiers": [ + "gate_epuf_fill", + "epuf_career_fill", + "2026-10-03-epuf-career-fill" + ], "lane": "EPUF career fill (locked gate)", "status": "used", "source": "SSA 2006 Earnings Public-Use File (EPUF)", - "unit_kind": "person_partition", "units": { - "partition": "EPUF2006_TEST_by_salted_person_hash" + "partition": "EPUF2006_TEST_by_salted_person_hash", + "earnings_years": "1951-2006" }, "unit_keys": [ - "ssa_epuf_2006:partition:epuf2006_test_by_salted_person_hash" + "ssa_epuf:2006:test_partition" ], "evidence": [ - "gates.yaml:5918 (holdout_basis)" + "gates.yaml:5918 (holdout_basis)", + "src/populace_dynamics/data/epuf.py:15 (capped taxable earnings, 1951-2006)" ], "scored_evidence": [ - "runs/epuf_fill_gate_test_v1.json" + { + "path": "runs/epuf_fill_gate_test_v1.json", + "field": "registration_id", + "equals": "2026-10-03-epuf-career-fill" + } ], "disclosed_overlaps": [], + "overlap_review": [], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; open_population.md rev 2 plans gate_young_earn on a new salt, disjoint from this partition" + "note": "seeded from gates.yaml at 1e739d32; open_population.md rev 2 plans gate_young_earn on a new salt of the same file, which shares this unit and must disclose it" } ] }, { "id": "H004", + "provenance": "historical_import", "gate": "gate_m4", + "gate_identifiers": [ + "gate_m4", + "m4_disability" + ], "lane": "M4 disability (locked gate)", "status": "used", "source": "PSID cross-year individual file employment status; SSA Annual Statistical Report on the SSDI Program, 2023", - "unit_kind": "publication_edition_tables", "units": { - "holdout_basis": [ - "ind2023er_employment_status", - "di_asr_2023" + "holdout": "person-disjoint side-A half of each of 100 split seeds over the work-limitation panel", + "employment_status_waves": "1982, 1983, 1993-1997 annual, 1999-2023 biennial", + "di_asr_2023_tables": [ + 19, + 35, + 36, + 49, + 50 ] }, "unit_keys": [ - "psid:ind2023er:employment_status", + "psid:1982:employment_status", + "psid:1983:employment_status", + "psid:1993-1997:employment_status", + "psid:1999-2023:employment_status", "ssa_di_asr:2023" ], "evidence": [ - "gates.yaml:2945 (holdout_basis)" + "gates.yaml:2945 (holdout_basis)", + "gates.yaml:2946-2953 (data_staged)", + "src/populace_dynamics/data/disability.py:19-24 (wave coverage)" ], "scored_evidence": [ - "runs/gate_m4_hazard_v1.json" + { + "path": "runs/gate_m4_hazard_v1.json", + "field": "gate", + "equals": "gate_m4" + } ], "disclosed_overlaps": [ + "H001", + "H002", "H005" ], + "overlap_review": [ + { + "id": "H003", + "reason": "EPUF 2006 is a person-level extract of earnings through 2006 (src/populace_dynamics/data/epuf.py:15); gate_m4 holds out tables of the DI ASR's 2023 edition. They share no edition or file year. Whether a DI ASR 2023 table reports data years before 2007 is for B0.2's inventory to itemize." + } + ], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; the edition's tables and the employment-status years are itemized by B0.2's inventory; shares di_asr_2023 with gate_w1 (H005)" + "note": "seeded from gates.yaml at 1e739d32; its PSID panel shares waves with gate_m6 (H001) and B2 (H002), and the DI ASR 2023 edition with gate_w1 (H005)" } ] }, { "id": "H005", + "provenance": "historical_import", "gate": "gate_w1", + "gate_identifiers": [ + "gate_w1", + "w1_representative_frame_transport" + ], "lane": "W1 frame transport (locked gate)", "status": "used", - "source": "SSA Annual Statistical Supplement 2023, section 6.B; SSA Annual Statistical Report on the SSDI Program, 2023; Census CPS tables; the certified 2024 frame", - "unit_kind": "publication_edition_tables", + "source": "SSA Annual Statistical Supplement 2023, section 6.B; SSA Annual Statistical Report on the SSDI Program, 2023; Census CPS tables; the certified 2024 frame; Mermin and Smith orderings", "units": { "holdout_basis": [ "populace_us_2024_certified", @@ -192,36 +317,59 @@ "DI ASR 2023 table 19", "Census HH-4, reference year 2023", "Census AD-3, reference year 2023" - ] + ], + "note": "whether each named table was gated or report-only is itemized by B0.2's inventory" }, "unit_keys": [ "ssa_supplement:2023:6.b5.1", "ssa_supplement:2023:6.b4", "ssa_supplement:2023:6.b3", "ssa_di_asr:2023:table_19", - "census_cps:2023:hh-4", - "census_cps:2023:ad-3", - "populace_us_2024:certified", - "mermin_smith:committed_orderings" + "census_cps_tables:2023:hh-4", + "census_cps_tables:2023:ad-3", + "populace_us_frame:2024:certified", + "mermin_smith_anchors:undated" ], "evidence": [ "gates.yaml:3735-3736 (holdout_basis)", "gates.yaml:4582-4584 (sources)" ], "scored_evidence": [ - "runs/gate_w1_candidate1_v1.json", - "runs/gate_w1_candidate2_v1.json", - "runs/gate_w1_candidate3_v1.json", - "runs/gate_w1_candidate4_v1.json" + { + "path": "runs/gate_w1_candidate1_v1.json", + "field": "gate", + "equals": "gate_w1" + }, + { + "path": "runs/gate_w1_candidate2_v1.json", + "field": "gate", + "equals": "gate_w1" + }, + { + "path": "runs/gate_w1_candidate3_v1.json", + "field": "gate", + "equals": "gate_w1" + }, + { + "path": "runs/gate_w1_candidate4_v1.json", + "field": "gate", + "equals": "gate_w1" + } ], "disclosed_overlaps": [ "H004" ], + "overlap_review": [ + { + "id": "H003", + "reason": "EPUF 2006 is a person-level extract of earnings through 2006 (src/populace_dynamics/data/epuf.py:15); gate_w1 used tables of the Supplement's and the DI ASR's 2023 editions. They share no edition or file year. The data years those tables report are for B0.2's inventory to itemize." + } + ], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; whether each named table was gated or report-only is itemized by B0.2's inventory; shares di_asr_2023 with gate_m4 (H004)" + "note": "seeded from gates.yaml at 1e739d32; shares the DI ASR 2023 edition with gate_m4 (H004)" } ] } diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json index 661eeb88a..e2d749c8c 100644 --- a/docs/design/track_b_holdout_registry_mutations.json +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -1,97 +1,226 @@ { "schema_version": 1, "verification_class": "accounting conformance", - "admitted_scope": "The held-out data registry's rules on invented registries and on the committed registry; no population, behavioral or forecast claim.", - "mutation_method": "One deliberate production-line change per check; the registry test module is expected to fail; original bytes restored before the next check. No mutated implementation is committed.", + "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", + "mutation_method": "One deliberate production-line change per check; the full registry test module runs and is expected to fail; original bytes restored before the next check. No mutated implementation is committed.", "restored_source_sha256": { - "src/populace_dynamics/track_b/holdout_registry.py": "6bfc2ce8beaccb837be050a51e73c051b767353399217667b0bc9ebf55bb1c63" + "src/populace_dynamics/track_b/holdout_registry.py": "599e8966fd75801ee1f3f45af3eb4a8aa06b4d2c017cec898e7d58cb19f55f5b" }, "checks": [ { - "mutation": "prefix overlap -> exact equality", - "old": "return left[:n] == right[:n]", - "new": "return left == right", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "mutation": "scope segments unchecked", + "old": "if not all(SCOPE_RX.fullmatch(part) for part in parts[2:]):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2020::x]", + "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2020:Upper]" + ] + }, + { + "mutation": "reversed year range accepted", + "old": "if end < start:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2022-2020]" + ] + }, + { + "mutation": "different sources overlap", + "old": "if left is None or right is None or left[0] != right[0]:", + "new": "if left is None or right is None:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", + "tests/test_track_b_holdout_registry.py::test_the_committed_registry_keeps_its_rules" + ] + }, + { + "mutation": "undated overlaps nothing", + "old": "if left[1] is None or right[1] is None:\n return True", + "new": "if left[1] is None or right[1] is None:\n return False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years" + ] + }, + { + "mutation": "ranges overlap only when equal", + "old": "return left[1][0] <= right[1][1] and right[1][0] <= left[1][1]", + "new": "return left[1] == right[1]", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_the_shared_di_report_edition_is_disclosed_both_ways" + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", + "tests/test_track_b_holdout_registry.py::test_the_seeded_shared_units_are_disclosed_both_ways[H001-H002]", + "tests/test_track_b_holdout_registry.py::test_the_seeded_shared_units_are_disclosed_both_ways[H001-H004]", + "tests/test_track_b_holdout_registry.py::test_the_seeded_shared_units_are_disclosed_both_ways[H002-H004]" ] }, { - "mutation": "history dates may go back", - "old": "elif date < previous_date:", + "mutation": "evidence may name another gate", + "old": "if equals not in identifiers:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_name_this_gate" + ] + }, + { + "mutation": "evidence paths may leave the repository", + "old": "if path.startswith((\"/\", \"~\")) or \"..\" in path.split(\"/\"):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + ] + }, + { + "mutation": "evidence need not be committed", + "old": "if path not in tracked:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + ] + }, + { + "mutation": "evidence may be a symlink", + "old": "if target.is_symlink() or not resolved.is_relative_to(root):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + ] + }, + { + "mutation": "evidence need not identify the gate", + "old": "if lookup(document, field) != equals:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + ] + }, + { + "mutation": "keys may name undeclared sources", + "old": "elif parsed[0] not in sources:", "new": "elif False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_history_dates_never_go_back" + "tests/test_track_b_holdout_registry.py::test_a_key_naming_no_source_is_refused" ] }, { - "mutation": "any transition allowed", + "mutation": "any first status", + "old": "if history[0][\"status\"] not in first:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses1-candidate-starts as one of]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses0-frozen-starts as one of]" + ] + }, + { + "mutation": "dates not parsed", + "old": "date = dt.date.fromisoformat(item[\"date\"])", + "new": "date = previous_date", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-10-01]", + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-99-99]", + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[yesterday]" + ] + }, + { + "mutation": "dates may go back", + "old": "if previous_date is not None and date < previous_date:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-10-01]" + ] + }, + { + "mutation": "any transition", "old": "status not in TRANSITIONS[previous_status]", "new": "False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[statuses0-used-not" + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses3-frozen-not a legal]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses2-used-not a legal]" ] }, { "mutation": "status need not match history", "old": "if entry[\"status\"] != previous_status:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[statuses3-candidate-is" + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses4-candidate-last status]" ] }, { "mutation": "used needs no scored evidence", - "old": "if not entry[\"scored_evidence\"]:", + "old": "if entry[\"status\"] == \"used\" and not entry[\"scored_evidence\"]:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_used_entry_needs_committed_scored_evidence" + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_name_this_gate" ] }, { - "mutation": "scored evidence need not exist", - "old": "if not _relative_file(repository, relative):", + "mutation": "alias sources accepted", + "old": "if catalog in catalogs:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_used_entry_needs_committed_scored_evidence" + "tests/test_track_b_holdout_registry.py::test_an_alias_source_is_refused" ] }, { "mutation": "complete needs no record", - "old": "and registry[\"complete\"] is True", + "old": "and registry[\"complete\"]", "new": "and False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_complete_needs_its_completion_record" ] }, { - "mutation": "repeated ids allowed", - "old": "if len(set(ids)) != len(ids):", + "mutation": "one identifier, two gates", + "old": "if owner != entry[\"gate\"]:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_repeated_ids_are_refused" + "tests/test_track_b_holdout_registry.py::test_repeated_ids_and_shared_gate_identifiers_are_refused" ] }, { - "mutation": "one-way disclosure allowed", + "mutation": "one-way disclosure accepted", "old": "elif entry[\"id\"] not in by_id[other][\"disclosed_overlaps\"]:", "new": "elif False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_one_way_disclosure_is_refused" @@ -101,47 +230,93 @@ "mutation": "withdrawn entries counted live", "old": "live = [entry for entry in entries if entry[\"status\"] != \"withdrawn\"]", "new": "live = list(entries)", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_invented_registries_that_disclose_every_overlap_are_valid" + "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen", + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", + "tests/test_track_b_holdout_registry.py::test_appending_lone_candidates_keeps_every_rule", + "tests/test_track_b_holdout_registry.py::test_settled_registries_are_valid" ] }, { - "mutation": "silent shared unit allowed", - "old": "other[\"id\"] not in entry[\"disclosed_overlaps\"]", - "new": "False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "mutation": "silent shared unit accepted", + "old": "other[\"id\"] not in entry[\"disclosed_overlaps\"]\n ):", + "new": "False\n ):", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused" ] }, + { + "mutation": "reviews may name later entries", + "old": "if position.get(name, index) >= index:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_must_name_an_earlier_entry" + ] + }, + { + "mutation": "family relations need no review", + "old": "and other[\"id\"] not in reviewed", + "new": "and False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused" + ] + }, + { + "mutation": "rules may shrink", + "old": "if not _extends(base.get(\"rules\", []), current.get(\"rules\", [])):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" + ] + }, + { + "mutation": "sources may change", + "old": "if sources.get(source) != declared:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" + ] + }, { "mutation": "complete may revert", "old": "if base.get(\"complete\") is True and current.get(\"complete\") is not True:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_rules_and_completeness_only_move_forward" + "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" ] }, { "mutation": "entries may be removed or reordered", "old": "if new_ids[: len(old_ids)] != old_ids:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused" + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_any_json_in_any_field_is_reported_not_raised", + "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused", + "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" ] }, { "mutation": "immutable fields may change", "old": "if after.get(key) != before.get(key):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused" @@ -151,20 +326,62 @@ "mutation": "extending lists may shrink", "old": "if not _extends(before.get(key, []), after.get(key, [])):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_extending_lists_never_shrink" ] }, { - "mutation": "absent base raises instead of None", - "old": "if listed.returncode == 0 and not listed.stdout.strip():", + "mutation": "imports allowed after completion", + "old": "if base.get(\"complete\") is True:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_close_when_the_registry_is_complete" + ] + }, + { + "mutation": "registrations may arrive frozen", + "old": "elif [item.get(\"status\") for item in entry.get(\"history\", [])] != [", + "new": "elif False and [item.get(\"status\") for item in entry.get(\"history\", [])] != [", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen" + ] + }, + { + "mutation": "push ignores its before commit", + "old": "if set(before) != {\"0\"} and _commit(repository, before):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit" + ] + }, + { + "mutation": "base branch compares with itself", + "old": "if base == head:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_no_base_raises_rather_than_passing", + "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" + ] + }, + { + "mutation": "absent base registry raises", + "old": "if not listed.stdout.strip():", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -x -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_the_base_registry_is_read_at_the_merge_base" + "tests/test_track_b_holdout_registry.py::test_a_pull_request_compares_with_its_merge_base" ] } ] diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index 6630f74dd..74be60495 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -323,6 +323,10 @@ # by B2's floor builder, comparator and scorer; nothing in the # historical projection imports it. Path("src/populace_dynamics/track_b/b2.py"), + # The held-out data registry's checks read only the registry, git and + # committed run artifacts; nothing in the historical projection + # imports them. + Path("src/populace_dynamics/track_b/holdout_registry.py"), # U2 documentary metadata stays outside the historical projection. Path("src/populace_dynamics/data/u2_source_registry.py"), Path("scripts/capture_track_u2_income_wealth.py"), diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py index 19d20ae38..903885ca4 100644 --- a/src/populace_dynamics/track_b/holdout_registry.py +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -1,31 +1,51 @@ """The held-out data registry and the rules every commit must keep. -``docs/design/track_b_holdout_registry.json`` records the source years, -editions, tables and partitions that Track B's gates and the -open-population lane's gates hold out. The dynamics coordinator made it -the single registry for every lane on 2026-10-21. This module enforces -its rules: +``docs/design/track_b_holdout_registry.json`` records the data that Track +B's gates and the open-population lane's gates hold out. The dynamics +coordinator made it the single registry for every lane on 2026-10-21. +This module enforces its rules: -- one schema, ``track_b_holdout_registry.v1``; +- one schema, ``track_b_holdout_registry.v1``, with every field typed; +- a declared source vocabulary: each source has a family, a title and a + catalog reference that no other source shares, so one source cannot be + registered under two names; +- unit keys ``source:years[:scope...]``, where ``years`` is a year, an + inclusive year range, or ``undated``. Two keys name a shared unit when + they have the same source and their years intersect (``undated`` + intersects every year). Scope segments describe the unit and never + narrow the overlap; +- no shared unit between live entries goes undisclosed, and every + disclosure is symmetric; +- relations the mechanical rule cannot see are reviewed by hand: an entry + lists, with a reason, every earlier live entry of the same source + family that it neither shares a unit with nor discloses; - unique entry ids, which are never removed, reordered or reused; -- no unit held out by two live entries silently: entries whose unit keys - overlap list each other in ``disclosed_overlaps``, and every disclosure - is symmetric; -- each entry's status is its history's last status, reached through legal - transitions only; -- a ``used`` entry names committed artifacts of the gate's scored run; +- histories with legal transitions only. A registration starts as a + ``candidate``, and is added in a commit of its own before it may move + on, so every registration reaches the base branch before its gate + freezes. A historical import, a gate that froze or scored before the + registry existed, starts as ``frozen`` or ``used`` and may be added only + while the registry is incomplete; +- a ``used`` entry cites committed, gate-identified artifacts of its + scored run; - entries only grow: against the base registry, an entry's identity and - units never change, and its history, evidence, scored evidence and - disclosures only extend; -- the registry stays marked incomplete until its completion record, B0.2's + units never change, and its lists only extend; +- the registry stays incomplete until its completion record, B0.2's mechanical exposure inventory, exists. +The overlap rule is deliberately coarse. It catches two gates that read +the same source at the same year, whatever variable or table they use. +It cannot see that two sources share underlying data, or that two +editions of a publication report the same data years. That is what the +family review is for. + Each check returns a list of problems; an empty list means the registry keeps the rules. """ from __future__ import annotations +import datetime as dt import json import os import re @@ -37,51 +57,63 @@ SCHEMA = "track_b_holdout_registry.v1" REGISTRY = "docs/design/track_b_holdout_registry.json" STATUSES = ("candidate", "frozen", "used", "withdrawn") -#: Legal next statuses. A first history item may hold any status, so prior -#: uses are recorded as ``used`` when they are entered. TRANSITIONS = { "candidate": frozenset({"frozen", "withdrawn"}), "frozen": frozenset({"used", "withdrawn"}), "used": frozenset(), "withdrawn": frozenset(), } -UNIT_KINDS = ( - "reference_years", - "publication_edition_tables", - "person_partition", -) +#: Each provenance's permitted first statuses. A historical import records +#: a gate that froze or scored before the registry existed. +FIRST_STATUS = { + "registration": frozenset({"candidate"}), + "historical_import": frozenset({"frozen", "used"}), +} +TEXT_FIELDS = ("schema", "title", "purpose", "completion_record") TOP_LEVEL = ( - "schema", - "title", - "purpose", + *TEXT_FIELDS, "rules", "status_values", + "sources", "complete", - "completion_record", "entries", ) -ENTRY_KEYS = ( +SOURCE_FIELDS = ("family", "title", "catalog", "key_year") +ENTRY_TEXT = ("id", "provenance", "gate", "lane", "source", "status") +ENTRY_LISTS = ( + "gate_identifiers", + "unit_keys", + "evidence", + "scored_evidence", + "disclosed_overlaps", + "overlap_review", + "history", +) +ENTRY_KEYS = (*ENTRY_TEXT, "units", *ENTRY_LISTS) +#: Fields that identify an entry and its units; they never change. +IMMUTABLE = ( "id", + "provenance", "gate", "lane", - "status", "source", - "unit_kind", "units", "unit_keys", +) +#: Lists that may only extend. +EXTENDING = ( + "gate_identifiers", "evidence", "scored_evidence", "disclosed_overlaps", + "overlap_review", "history", ) -#: Fields that identify an entry and its units; they never change. -IMMUTABLE = ("id", "gate", "lane", "source", "unit_kind", "units", "unit_keys") -#: Lists that may only extend. -EXTENDING = ("evidence", "scored_evidence", "disclosed_overlaps", "history") ID_RX = re.compile(r"H[0-9]{3,}") -SEGMENT = r"[a-z0-9][a-z0-9_.\-]*" -UNIT_KEY_RX = re.compile(rf"{SEGMENT}(?::{SEGMENT})*") -DATE_RX = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}") +SOURCE_RX = re.compile(r"[a-z][a-z0-9_]*") +YEARS_RX = re.compile(r"(?P[0-9]{4})(?:-(?P[0-9]{4}))?|undated") +SCOPE_RX = re.compile(r"[a-z0-9][a-z0-9_.\-]*") +SHA_RX = re.compile(r"[0-9a-f]{40}") def load(repository: Path) -> dict[str, Any]: @@ -89,13 +121,41 @@ def load(repository: Path) -> dict[str, Any]: return json.loads((repository / REGISTRY).read_text(encoding="utf-8")) +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def parse_key(key: Any) -> tuple[str, tuple[int, int] | None] | None: + """``(source, years)`` for a well-formed unit key, with ``years`` an + inclusive range or ``None`` for ``undated``; ``None`` otherwise.""" + if not isinstance(key, str): + return None + parts = key.split(":") + if len(parts) < 2 or not SOURCE_RX.fullmatch(parts[0]): + return None + if not all(SCOPE_RX.fullmatch(part) for part in parts[2:]): + return None + years = YEARS_RX.fullmatch(parts[1]) + if years is None: + return None + if parts[1] == "undated": + return parts[0], None + start = int(years["start"]) + end = int(years["end"] or start) + if end < start: + return None + return parts[0], (start, end) + + def keys_overlap(a: str, b: str) -> bool: - """Whether two unit keys name a shared unit: one is the other or a - ``:``-segment prefix of it, so ``ssa_di_asr:2023`` (an edition) - overlaps ``ssa_di_asr:2023:table_19`` (one of its tables).""" - left, right = a.split(":"), b.split(":") - n = min(len(left), len(right)) - return left[:n] == right[:n] + """Whether two well-formed unit keys name a shared unit: the same + source, with intersecting years.""" + left, right = parse_key(a), parse_key(b) + if left is None or right is None or left[0] != right[0]: + return False + if left[1] is None or right[1] is None: + return True + return left[1][0] <= right[1][1] and right[1][0] <= left[1][1] def entries_overlap(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: @@ -105,70 +165,173 @@ def entries_overlap(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: ) -def _relative_file(repository: Path, relative: Any) -> bool: - """``relative`` names a file inside ``repository``.""" - if not isinstance(relative, str) or not relative: - return False - if relative.startswith(("/", "~")) or ".." in relative.split("/"): - return False - return (repository / relative).is_file() +def families(entry: Mapping[str, Any], sources: Mapping[str, Any]) -> set[str]: + """The source families an entry's unit keys belong to.""" + return { + sources[parsed[0]]["family"] + for parsed in map(parse_key, entry["unit_keys"]) + if parsed is not None and parsed[0] in sources + } -def _entry_problems( - entry: Mapping[str, Any], repository: Path | None +def lookup(document: Any, dotted: str) -> Any: + """``document``'s value at a dotted field path, or a sentinel.""" + for part in dotted.split("."): + if not isinstance(document, Mapping) or part not in document: + return _MISSING + document = document[part] + return document + + +_MISSING = object() + + +def tracked_files(repository: Path) -> set[str] | None: + """Paths git tracks in ``repository``, or ``None`` without git.""" + listed = _git(repository, "ls-files", "-z") + if listed.returncode != 0: + return None + return {path for path in listed.stdout.split("\0") if path} + + +def _evidence_problems( + name: str, + item: Mapping[str, Any], + identifiers: Sequence[str], + repository: Path | None, + tracked: set[str] | None, ) -> list[str]: - """Problems inside one entry, without reference to the others.""" - name = entry.get("id", "") + """Problems with one scored-evidence citation.""" + path, field, equals = item["path"], item["field"], item["equals"] + if equals not in identifiers: + return [f"{name}: scored evidence names {equals!r}, not this gate"] + if repository is None: + return [] + if path.startswith(("/", "~")) or ".." in path.split("/"): + return [f"{name}: scored evidence {path!r} is outside the repository"] + if tracked is None: + return [f"{name}: cannot list committed files to check {path!r}"] + if path not in tracked: + return [f"{name}: scored evidence {path!r} is not committed"] + target = repository / path + root = repository.resolve() + resolved = target.resolve() + if target.is_symlink() or not resolved.is_relative_to(root): + return [ + f"{name}: scored evidence {path!r} is a symlink or leaves " + "the repository" + ] + if not resolved.is_file(): + return [f"{name}: scored evidence {path!r} is not a file"] + try: + document = json.loads(resolved.read_text(encoding="utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return [f"{name}: scored evidence {path!r} is not JSON"] + if lookup(document, field) != equals: + return [ + f"{name}: scored evidence {path!r} does not have " + f"{field} == {equals!r}" + ] + return [] + + +def _shape_problems(entry: Any) -> list[str]: + """Type problems in one entry; later checks assume none.""" + if not isinstance(entry, Mapping): + return ["registry: an entry is not a mapping"] + name = entry.get("id") if _text(entry.get("id")) else "" missing = [key for key in ENTRY_KEYS if key not in entry] if missing: return [f"{name}: missing {', '.join(missing)}"] - problems: list[str] = [] - if not isinstance(name, str) or not ID_RX.fullmatch(name): - problems.append(f"{name}: id must match H followed by 3+ digits") - for key in ("gate", "lane", "source"): - if not isinstance(entry[key], str) or not entry[key]: - problems.append(f"{name}: {key} must be a non-empty string") - if entry["unit_kind"] not in UNIT_KINDS: - problems.append(f"{name}: unknown unit_kind {entry['unit_kind']!r}") + problems = [ + f"{name}: {key} must be a non-empty string" + for key in ENTRY_TEXT + if not _text(entry[key]) + ] + problems += [ + f"{name}: {key} must be a list" + for key in ENTRY_LISTS + if not isinstance(entry[key], list) + ] if not isinstance(entry["units"], Mapping) or not entry["units"]: problems.append(f"{name}: units must be a non-empty mapping") - keys = entry["unit_keys"] - if ( - not isinstance(keys, list) - or not keys - or not all( - isinstance(k, str) and UNIT_KEY_RX.fullmatch(k) for k in keys - ) + if problems: + return problems + for key in ( + "gate_identifiers", + "unit_keys", + "evidence", + "disclosed_overlaps", ): - problems.append(f"{name}: unit_keys must be well-formed keys") - elif len(set(keys)) != len(keys): - problems.append(f"{name}: unit_keys repeat") - for key in ("evidence", "scored_evidence", "disclosed_overlaps"): - if not isinstance(entry[key], list) or not all( - isinstance(item, str) and item for item in entry[key] - ): - problems.append(f"{name}: {key} must be a list of strings") + if not all(_text(item) for item in entry[key]): + problems.append(f"{name}: {key} must hold non-empty strings") + shapes = { + "scored_evidence": ("path", "field", "equals"), + "overlap_review": ("id", "reason"), + "history": ("date", "status", "note"), + } + for key, fields in shapes.items(): + for item in entry[key]: + if not isinstance(item, Mapping) or not all( + _text(item.get(field)) for field in fields + ): + problems.append( + f"{name}: each {key} item needs non-empty " + f"{', '.join(fields)}" + ) + break + return problems + + +def _entry_problems( + entry: Mapping[str, Any], + sources: Mapping[str, Any], + repository: Path | None, + tracked: set[str] | None, +) -> list[str]: + """Problems inside one well-shaped entry.""" + name = entry["id"] + problems: list[str] = [] + if not ID_RX.fullmatch(name): + problems.append(f"{name}: id must be H followed by 3+ digits") + if entry["provenance"] not in FIRST_STATUS: + problems.append(f"{name}: unknown provenance") + return problems + for key in ("gate_identifiers", "unit_keys", "evidence"): + if not entry[key]: + problems.append(f"{name}: {key} must not be empty") + elif len(set(entry[key])) != len(entry[key]): + problems.append(f"{name}: {key} repeat") + for key in entry["unit_keys"]: + parsed = parse_key(key) + if parsed is None: + problems.append(f"{name}: malformed unit key {key!r}") + elif parsed[0] not in sources: + problems.append(f"{name}: unit key {key!r} names no source") history = entry["history"] - if not isinstance(history, list) or not history: - problems.append(f"{name}: history must be a non-empty list") + if not history: + problems.append(f"{name}: history must not be empty") return problems - previous_status, previous_date = None, "" + first = FIRST_STATUS[entry["provenance"]] + if history[0]["status"] not in first: + problems.append( + f"{name}: a {entry['provenance']} starts as one of " + f"{sorted(first)}, not {history[0]['status']!r}" + ) + previous_status, previous_date = None, None for index, item in enumerate(history): - if not isinstance(item, Mapping): - problems.append(f"{name}: history[{index}] is not a mapping") - return problems - status, date = item.get("status"), item.get("date") + status = item["status"] if status not in STATUSES: problems.append(f"{name}: history[{index}] status {status!r}") return problems - if not isinstance(date, str) or not DATE_RX.fullmatch(date): - problems.append(f"{name}: history[{index}] date {date!r}") - elif date < previous_date: + try: + date = dt.date.fromisoformat(item["date"]) + except ValueError: + problems.append(f"{name}: history[{index}] date is not a date") + date = previous_date + if previous_date is not None and date < previous_date: problems.append(f"{name}: history[{index}] goes back in time") - else: - previous_date = date - if not isinstance(item.get("note"), str) or not item["note"]: - problems.append(f"{name}: history[{index}] needs a note") + previous_date = date if previous_status is not None and ( status not in TRANSITIONS[previous_status] ): @@ -182,65 +345,111 @@ def _entry_problems( f"{name}: status {entry['status']!r} is not its history's " f"last status {previous_status!r}" ) - if entry["status"] == "used": - if not entry["scored_evidence"]: - problems.append(f"{name}: a used entry needs scored_evidence") - elif repository is not None: - for relative in entry["scored_evidence"]: - if not _relative_file(repository, relative): - problems.append( - f"{name}: scored_evidence {relative!r} is not a " - "committed file" - ) + if entry["status"] == "used" and not entry["scored_evidence"]: + problems.append(f"{name}: a used entry needs scored_evidence") + for item in entry["scored_evidence"]: + problems.extend( + _evidence_problems( + name, item, entry["gate_identifiers"], repository, tracked + ) + ) return problems -def validate( - registry: Mapping[str, Any], repository: Path | None = None -) -> list[str]: - """Every rule that one registry must keep on its own. With - ``repository``, scored evidence and the completion record are checked - against its files.""" +def _vocabulary_problems(sources: Any) -> list[str]: + if not isinstance(sources, Mapping) or not sources: + return ["registry: sources must be a non-empty mapping"] + problems: list[str] = [] + catalogs: dict[str, str] = {} + for source, declared in sources.items(): + if not isinstance(source, str) or not SOURCE_RX.fullmatch(source): + problems.append(f"source {source!r}: malformed id") + if not isinstance(declared, Mapping) or not all( + _text(declared.get(field)) for field in SOURCE_FIELDS + ): + problems.append( + f"source {source!r}: needs {', '.join(SOURCE_FIELDS)}" + ) + continue + catalog = " ".join(declared["catalog"].split()).casefold() + if catalog in catalogs: + problems.append( + f"source {source!r}: shares its catalog with " + f"{catalogs[catalog]!r}, so one of them is an alias" + ) + catalogs[catalog] = source + return problems + + +def validate(registry: Any, repository: Path | None = None) -> list[str]: + """Every rule one registry must keep on its own. With ``repository``, + scored evidence and the completion record are checked against its + committed files.""" + if not isinstance(registry, Mapping): + return ["registry: not a mapping"] missing = [key for key in TOP_LEVEL if key not in registry] if missing: return [f"registry: missing {', '.join(missing)}"] - problems: list[str] = [] + problems = [ + f"registry: {key} must be a non-empty string" + for key in TEXT_FIELDS + if not _text(registry[key]) + ] + if problems: + return problems if registry["schema"] != SCHEMA: problems.append(f"registry: schema {registry['schema']!r}") - if not isinstance(registry["rules"], list) or not all( - isinstance(rule, str) and rule for rule in registry["rules"] + rules = registry["rules"] + if not isinstance(rules, list) or not rules: + problems.append("registry: rules must be a non-empty list") + elif not all(_text(rule) for rule in rules): + problems.append("registry: each rule must be a non-empty string") + values = registry["status_values"] + if ( + not isinstance(values, Mapping) + or set(values) != set(STATUSES) + or not all(_text(text) for text in values.values()) ): - problems.append("registry: rules must be a list of strings") - if not isinstance(registry["status_values"], Mapping) or set( - registry["status_values"] - ) != set(STATUSES): - problems.append("registry: status_values must name each status") + problems.append("registry: status_values must describe each status") if not isinstance(registry["complete"], bool): problems.append("registry: complete must be true or false") + problems.extend(_vocabulary_problems(registry["sources"])) + entries = registry["entries"] + if not isinstance(entries, list): + problems.append("registry: entries must be a list") + if problems: + return problems + for entry in entries: + problems.extend(_shape_problems(entry)) + if problems: + return problems record = registry["completion_record"] - if not isinstance(record, str) or not record: - problems.append("registry: completion_record must name a path") - elif ( + if ( repository is not None - and registry["complete"] is True - and not _relative_file(repository, record) + and registry["complete"] + and not (repository / record).is_file() ): problems.append( f"registry: marked complete, but {record} does not exist" ) - entries = registry["entries"] - if not isinstance(entries, list): - return [*problems, "registry: entries must be a list"] + sources = registry["sources"] + tracked = tracked_files(repository) if repository is not None else None for entry in entries: - if not isinstance(entry, Mapping): - return [*problems, "registry: an entry is not a mapping"] - problems.extend(_entry_problems(entry, repository)) + problems.extend(_entry_problems(entry, sources, repository, tracked)) if problems: return problems ids = [entry["id"] for entry in entries] if len(set(ids)) != len(ids): - problems.append("registry: entry ids repeat") - return problems + return ["registry: entry ids repeat"] + owners: dict[str, str] = {} + for entry in entries: + for identifier in entry["gate_identifiers"]: + owner = owners.setdefault(identifier, entry["gate"]) + if owner != entry["gate"]: + problems.append( + f"{entry['id']}: gate identifier {identifier!r} " + f"already identifies {owner!r}" + ) by_id = {entry["id"]: entry for entry in entries} for entry in entries: for other in entry["disclosed_overlaps"]: @@ -263,12 +472,40 @@ def validate( f"{entry['id']} and {other['id']} hold out a shared " "unit without disclosing it" ) + position = {name: index for index, name in enumerate(ids)} + for index, entry in enumerate(entries): + reviewed = {item["id"] for item in entry["overlap_review"]} + for name in reviewed: + if position.get(name, index) >= index: + problems.append( + f"{entry['id']}: reviews {name}, which is not an " + "earlier entry" + ) + if entry["status"] == "withdrawn": + continue + mine = families(entry, sources) + for other in entries[:index]: + if ( + other["status"] != "withdrawn" + and mine & families(other, sources) + and not entries_overlap(entry, other) + and other["id"] not in entry["disclosed_overlaps"] + and other["id"] not in reviewed + ): + problems.append( + f"{entry['id']}: shares a source family with " + f"{other['id']} but neither discloses nor reviews it" + ) return problems -def _extends(base: Sequence[Any], current: Sequence[Any]) -> bool: - """``current`` keeps ``base`` as its prefix.""" - return list(current[: len(base)]) == list(base) +def _extends(base: Any, current: Any) -> bool: + """``current`` is a list that keeps the list ``base`` as its prefix.""" + return ( + isinstance(base, list) + and isinstance(current, list) + and current[: len(base)] == base + ) def append_only_problems( @@ -282,10 +519,20 @@ def append_only_problems( problems.append(f"registry: {key} changed") if not _extends(base.get("rules", []), current.get("rules", [])): problems.append("registry: rules may only be appended") + sources = current.get("sources") + if not isinstance(sources, Mapping): + sources = {} + for source, declared in base.get("sources", {}).items(): + if sources.get(source) != declared: + problems.append(f"source {source!r}: changed or removed") if base.get("complete") is True and current.get("complete") is not True: problems.append("registry: a complete registry became incomplete") old = base.get("entries", []) - new = current.get("entries", []) + new = current.get("entries") + if not isinstance(new, list) or not all( + isinstance(entry, Mapping) for entry in new + ): + return [*problems, "registry: entries are malformed"] old_ids = [entry.get("id") for entry in old] new_ids = [entry.get("id") for entry in new] if new_ids[: len(old_ids)] != old_ids: @@ -300,6 +547,21 @@ def append_only_problems( for key in EXTENDING: if not _extends(before.get(key, []), after.get(key, [])): problems.append(f"{before['id']}: {key} may only extend") + for entry in new[len(old) :]: + name = entry.get("id") + if entry.get("provenance") == "historical_import": + if base.get("complete") is True: + problems.append( + f"{name}: historical imports close when the registry " + "is complete" + ) + elif [item.get("status") for item in entry.get("history", [])] != [ + "candidate" + ]: + problems.append( + f"{name}: a new registration is added as a lone candidate, " + "and moves on only in a later commit" + ) return problems @@ -312,23 +574,62 @@ def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: ) -def base_registry(repository: Path) -> dict[str, Any] | None: - """The registry at the merge base of ``HEAD`` and the base branch - (``GITHUB_BASE_REF`` on a pull request, otherwise ``master``), or - ``None`` when the base has no registry yet. Raises ``LookupError`` - when no base can be resolved.""" +def _commit(repository: Path, ref: str) -> str | None: + found = _git( + repository, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}" + ) + return found.stdout.strip() if found.returncode == 0 else None + + +def base_commit(repository: Path) -> str: + """The commit the registry is compared against. + + - On a push (``GITHUB_EVENT_NAME=push``), the event's ``before`` + commit, which was the branch tip before the push. + - Otherwise, the merge base of ``HEAD`` and the base branch + (``GITHUB_BASE_REF`` on a pull request, otherwise ``master``), + preferring its ``origin/`` ref. + - When that merge base is ``HEAD`` itself, as on the base branch, its + first parent. + + Raises ``LookupError`` when none of these resolves.""" + head = _commit(repository, "HEAD") + if head is None: + raise LookupError(f"no HEAD in {repository}") + if os.environ.get("GITHUB_EVENT_NAME") == "push": + event = os.environ.get("GITHUB_EVENT_PATH") + before = None + if event and Path(event).is_file(): + before = json.loads(Path(event).read_text()).get("before") + if isinstance(before, str) and SHA_RX.fullmatch(before): + if set(before) != {"0"} and _commit(repository, before): + return before branch = os.environ.get("GITHUB_BASE_REF") or "master" for ref in (f"origin/{branch}", branch): - merge_base = _git(repository, "merge-base", "HEAD", ref) - if merge_base.returncode == 0: + found = _git(repository, "merge-base", "HEAD", ref) + if found.returncode == 0: + base = found.stdout.strip() break else: raise LookupError(f"no merge base with {branch} in {repository}") - commit = merge_base.stdout.strip() + if base == head: + parent = _commit(repository, "HEAD^1") + if parent is None: + raise LookupError("HEAD is its own base and has no parent") + return parent + return base + + +def base_registry(repository: Path) -> dict[str, Any] | None: + """The registry at :func:`base_commit`, or ``None`` when the base has + no registry yet. Raises ``LookupError`` when no base resolves.""" + commit = base_commit(repository) + listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) + if listed.returncode != 0: + raise LookupError(f"cannot list {REGISTRY} at {commit}") + if not listed.stdout.strip(): + return None shown = _git(repository, "show", f"{commit}:{REGISTRY}") if shown.returncode != 0: - listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) - if listed.returncode == 0 and not listed.stdout.strip(): - return None raise LookupError(f"cannot read {REGISTRY} at {commit}") return json.loads(shown.stdout) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index af067eacc..63357dd8f 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,9 +39,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 9,983 | +| `unit` | 10,020 | | `artifact` | 4,253 | | `integration_psid` | 1,450 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 308 | -| **Total** | **16,514** | +| **Total** | **16,551** | diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index e97b8dbc4..d387cbb2a 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -187,6 +187,7 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/track_b/arf.py"), Path("src/populace_dynamics/track_b/recompute.py"), Path("src/populace_dynamics/track_b/b2.py"), + Path("src/populace_dynamics/track_b/holdout_registry.py"), Path("src/populace_dynamics/data/u2_source_registry.py"), Path("scripts/capture_track_u2_income_wealth.py"), Path("scripts/capture_track_u2_ssi_parameters.py"), @@ -390,6 +391,7 @@ def test_post_review_exclusions_are_unreachable_from_birth_evidence(): "populace_dynamics.track_b.arf", "populace_dynamics.track_b.recompute", "populace_dynamics.track_b.b2", + "populace_dynamics.track_b.holdout_registry", } assert track_b_modules.issubset(module_paths) u2_registry = "populace_dynamics.data.u2_source_registry" diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py index 5ee2e0432..a8f58557d 100644 --- a/tests/test_track_b_holdout_registry.py +++ b/tests/test_track_b_holdout_registry.py @@ -1,11 +1,11 @@ """The held-out data registry keeps its rules on every commit. ``docs/design/track_b_holdout_registry.json`` is the single registry of -held-out years, editions, tables and partitions for Track B's gates and -the open-population lane's gates (dynamics coordinator, 2026-10-21). +the data that Track B's gates and the open-population lane's gates hold +out (dynamics coordinator, 2026-10-21). ``populace_dynamics.track_b.holdout_registry`` states the rules; these -tests run them on the committed registry, against its base branch, and -as properties over invented registries. +tests run them on the committed registry, against its base commit, and as +properties over invented registries. """ from __future__ import annotations @@ -24,11 +24,38 @@ ROOT = Path(__file__).resolve().parents[1] +#: Specified here, independently of the module, so that dropping a field +#: from the module's lists fails a test. +EXPECTED_IMMUTABLE = ( + "id", + "provenance", + "gate", + "lane", + "source", + "units", + "unit_keys", +) +EXPECTED_EXTENDING = ( + "gate_identifiers", + "evidence", + "scored_evidence", + "disclosed_overlaps", + "overlap_review", + "history", +) + def committed() -> dict: return registry.load(ROOT) +def by_id(current: dict) -> dict: + return {entry["id"]: entry for entry in current["entries"]} + + +# The committed registry ---------------------------------------------------- + + def test_the_committed_registry_keeps_its_rules(): assert registry.validate(committed(), ROOT) == [] @@ -39,7 +66,7 @@ def test_the_committed_registry_only_appends_to_its_base(): except LookupError as error: if os.environ.get("GITHUB_ACTIONS") == "true": pytest.fail(f"CI must resolve the base registry: {error}") - pytest.skip(f"no base branch here: {error}") + pytest.skip(f"no base commit here: {error}") if base is None: return assert registry.validate(base) == [] @@ -49,65 +76,87 @@ def test_the_committed_registry_only_appends_to_its_base(): def test_the_registry_stays_incomplete_until_the_inventory_lands(): current = committed() assert current["schema"] == registry.SCHEMA - record = ROOT / current["completion_record"] - if not record.is_file(): + if not (ROOT / current["completion_record"]).is_file(): assert current["complete"] is False assert any("incomplete" in rule for rule in current["rules"]) -def test_the_shared_di_report_edition_is_disclosed_both_ways(): - by_id = {entry["id"]: entry for entry in committed()["entries"]} - m4, w1 = by_id["H004"], by_id["H005"] - assert registry.entries_overlap(m4, w1) - assert "H005" in m4["disclosed_overlaps"] - assert "H004" in w1["disclosed_overlaps"] +@pytest.mark.parametrize( + "a, b", + [("H001", "H002"), ("H001", "H004"), ("H002", "H004"), ("H004", "H005")], +) +def test_the_seeded_shared_units_are_disclosed_both_ways(a, b): + entries = by_id(committed()) + assert registry.entries_overlap(entries[a], entries[b]) + assert b in entries[a]["disclosed_overlaps"] + assert a in entries[b]["disclosed_overlaps"] -def test_every_used_entry_names_committed_scored_artifacts(): - for entry in committed()["entries"]: - if entry["status"] == "used": - assert entry["scored_evidence"], entry["id"] - for relative in entry["scored_evidence"]: - assert (ROOT / relative).is_file(), relative +def test_the_field_lists_are_as_specified(): + assert registry.IMMUTABLE == EXPECTED_IMMUTABLE + assert registry.EXTENDING == EXPECTED_EXTENDING + assert set(EXPECTED_IMMUTABLE) | set(EXPECTED_EXTENDING) | { + "status", + } == set(registry.ENTRY_KEYS) -# Invented registries ------------------------------------------------------ +# Invented registries ------------------------------------------------------- +SOURCES = { + "s_a": {"family": "f1", "title": "A", "catalog": "cat a", "key_year": "y"}, + "s_b": {"family": "f1", "title": "B", "catalog": "cat b", "key_year": "y"}, + "s_c": {"family": "f2", "title": "C", "catalog": "cat c", "key_year": "y"}, +} KEY_POOL = ( - "alpha", - "alpha:2020", - "alpha:2021", - "alpha:2021:t1", - "beta:x", - "beta:y", - "beta:y:z", - "gamma:p", + "s_a:2020", + "s_a:2020-2022:x", + "s_a:2023", + "s_a:undated", + "s_b:2021:t1", + "s_b:2025", + "s_c:2020", + "s_c:2030-2031", ) PATHS = { - "candidate": ["candidate"], - "frozen": ["candidate", "frozen"], - "used": ["candidate", "frozen", "used"], - "withdrawn": ["candidate", "withdrawn"], - "seeded_used": ["used"], + ("registration", "candidate"): ["candidate"], + ("registration", "frozen"): ["candidate", "frozen"], + ("registration", "used"): ["candidate", "frozen", "used"], + ("registration", "withdrawn"): ["candidate", "withdrawn"], + ("historical_import", "used"): ["used"], + ("historical_import", "frozen"): ["frozen"], + ("historical_import", "frozen_used"): ["frozen", "used"], } -def make_entry(index: int, keys: list[str], path: str) -> dict: +def make_entry( + index: int, keys: list[str], path=("registration", "candidate") +): statuses = PATHS[path] + gate = f"gate_{index}" return { "id": f"H{index + 1:03d}", - "gate": f"gate_{index}", + "provenance": path[0], + "gate": gate, + "gate_identifiers": [gate], "lane": "invented", "status": statuses[-1], "source": "invented source", - "unit_kind": "reference_years", "units": {"years": [index]}, - "unit_keys": keys, + "unit_keys": list(keys), "evidence": [f"evidence {index}"], "scored_evidence": ( - [f"scores/score_{index}.txt"] if statuses[-1] == "used" else [] + [ + { + "path": f"scores/s{index}.json", + "field": "gate", + "equals": gate, + } + ] + if statuses[-1] == "used" + else [] ), "disclosed_overlaps": [], + "overlap_review": [], "history": [ {"date": f"2026-10-{20 + step:02d}", "status": s, "note": "n"} for step, s in enumerate(statuses) @@ -115,8 +164,9 @@ def make_entry(index: int, keys: list[str], path: str) -> dict: } -def disclose_all(entries: list[dict]) -> None: - """Disclose every overlap between live entries, both ways.""" +def settle(entries: list[dict], sources=SOURCES) -> None: + """Disclose every shared unit both ways, and review every unshared + same-family relation, as a careful registrant would.""" live = [entry for entry in entries if entry["status"] != "withdrawn"] for index, entry in enumerate(live): for other in live[index + 1 :]: @@ -125,16 +175,33 @@ def disclose_all(entries: list[dict]) -> None: entry["disclosed_overlaps"].append(other["id"]) if entry["id"] not in other["disclosed_overlaps"]: other["disclosed_overlaps"].append(entry["id"]) - - -def make_registry(entries: list[dict]) -> dict: + for index, entry in enumerate(entries): + if entry["status"] == "withdrawn": + continue + reviewed = {item["id"] for item in entry["overlap_review"]} + for other in entries[:index]: + if ( + other["status"] != "withdrawn" + and registry.families(entry, sources) + & registry.families(other, sources) + and not registry.entries_overlap(entry, other) + and other["id"] not in entry["disclosed_overlaps"] + and other["id"] not in reviewed + ): + entry["overlap_review"].append( + {"id": other["id"], "reason": "different years"} + ) + + +def make_registry(entries: list[dict], complete=False) -> dict: return { "schema": registry.SCHEMA, "title": "invented", "purpose": "invented", "rules": ["rule one", "the registry is incomplete"], "status_values": {status: status for status in registry.STATUSES}, - "complete": False, + "sources": copy.deepcopy(SOURCES), + "complete": complete, "completion_record": "docs/design/inventory.json", "entries": entries, } @@ -150,91 +217,164 @@ def make_registry(entries: list[dict]) -> dict: min_size=0, max_size=6, ) +registration_specs = st.lists( + st.lists(st.sampled_from(KEY_POOL), min_size=1, max_size=3, unique=True), + min_size=0, + max_size=4, +) -def build(specs) -> dict: +def build(specs, complete=False) -> dict: entries = [ make_entry(i, keys, path) for i, (keys, path) in enumerate(specs) ] - disclose_all(entries) - return make_registry(entries) + settle(entries) + return make_registry(entries, complete) -@settings(max_examples=150, deadline=None) +def shared_pairs(entries): + live = [e for e in entries if e["status"] != "withdrawn"] + return [ + (a, b) + for i, a in enumerate(live) + for b in live[i + 1 :] + if registry.entries_overlap(a, b) + ] + + +@settings(max_examples=200, deadline=None) @given(entry_specs) -def test_invented_registries_that_disclose_every_overlap_are_valid(specs): +def test_settled_registries_are_valid(specs): assert registry.validate(build(specs)) == [] -@settings(max_examples=150, deadline=None) +@settings(max_examples=200, deadline=None) @given(entry_specs, st.data()) def test_a_one_way_disclosure_is_refused(specs, data): current = build(specs) disclosing = [e for e in current["entries"] if e["disclosed_overlaps"]] if not disclosing: return - entry = data.draw(st.sampled_from(disclosing)) - entry["disclosed_overlaps"].pop() + data.draw(st.sampled_from(disclosing))["disclosed_overlaps"].pop() problems = registry.validate(current) - assert problems assert any( "does not disclose it back" in p or "without disclosing" in p for p in problems ) -@settings(max_examples=150, deadline=None) +@settings(max_examples=200, deadline=None) @given(entry_specs) def test_a_silent_shared_unit_is_refused(specs): current = build(specs) for entry in current["entries"]: entry["disclosed_overlaps"] = [] - live = [e for e in current["entries"] if e["status"] != "withdrawn"] - shared = any( - registry.entries_overlap(a, b) - for i, a in enumerate(live) - for b in live[i + 1 :] - ) + settle_reviews_only(current["entries"]) problems = registry.validate(current) - assert bool(problems) == shared - if shared: - assert any("without disclosing" in p for p in problems) + shared = bool(shared_pairs(current["entries"])) + assert any("without disclosing" in p for p in problems) == shared + + +def settle_reviews_only(entries): + for entry in entries: + entry["overlap_review"] = [] + for index, entry in enumerate(entries): + if entry["status"] == "withdrawn": + continue + for other in entries[:index]: + if ( + other["status"] != "withdrawn" + and registry.families(entry, SOURCES) + & registry.families(other, SOURCES) + and not registry.entries_overlap(entry, other) + ): + entry["overlap_review"].append( + {"id": other["id"], "reason": "different years"} + ) + + +@settings(max_examples=200, deadline=None) +@given(entry_specs) +def test_an_unreviewed_family_relation_is_refused(specs): + current = build(specs) + entries = current["entries"] + needed = any(entry["overlap_review"] for entry in entries) + for entry in entries: + entry["overlap_review"] = [] + problems = registry.validate(current) + assert any("neither discloses nor reviews" in p for p in problems) == ( + needed + ) -@settings(max_examples=150, deadline=None) -@given(entry_specs, entry_specs) -def test_appending_disclosed_entries_keeps_every_rule(base_specs, more): +@settings(max_examples=200, deadline=None) +@given(entry_specs, registration_specs) +def test_appending_lone_candidates_keeps_every_rule(base_specs, more): base = build(base_specs) current = copy.deepcopy(base) start = len(current["entries"]) - for offset, (keys, path) in enumerate(more): - current["entries"].append(make_entry(start + offset, keys, path)) - disclose_all(current["entries"]) + for offset, keys in enumerate(more): + current["entries"].append(make_entry(start + offset, keys)) + settle(current["entries"]) assert registry.validate(current) == [] assert registry.append_only_problems(base, current) == [] +@settings(max_examples=100, deadline=None) +@given( + entry_specs, + st.sampled_from( + [p for p in PATHS if p[0] == "registration" and p[1] != "candidate"] + ), +) +def test_a_registration_cannot_arrive_already_frozen(base_specs, path): + base = build(base_specs) + current = copy.deepcopy(base) + current["entries"].append( + make_entry(len(current["entries"]), ["s_c:2099"], path) + ) + settle(current["entries"]) + assert registry.validate(current) == [] + assert any( + "lone candidate" in p + for p in registry.append_only_problems(base, current) + ) + + +def test_imports_close_when_the_registry_is_complete(): + for complete in (False, True): + base = build([(["s_a:2020"], ("historical_import", "used"))], complete) + current = copy.deepcopy(base) + current["entries"].append( + make_entry(1, ["s_c:2099"], ("historical_import", "used")) + ) + problems = registry.append_only_problems(base, current) + assert bool(problems) == complete + + IMMUTABLE_EDITS = { "id": "H999", + "provenance": "registration", "gate": "gate_other", "lane": "other lane", "source": "other source", - "unit_kind": "person_partition", "units": {"years": [-1]}, - "unit_keys": ["delta:other"], + "unit_keys": ["s_c:2199"], } @settings(max_examples=150, deadline=None) @given( entry_specs.filter(bool), - st.sampled_from(registry.IMMUTABLE), + st.sampled_from(sorted(IMMUTABLE_EDITS)), st.data(), ) def test_changing_an_earlier_entry_is_refused(specs, key, data): base = build(specs) current = copy.deepcopy(base) entry = data.draw(st.sampled_from(current["entries"])) + if entry[key] == IMMUTABLE_EDITS[key]: + return entry[key] = IMMUTABLE_EDITS[key] assert registry.append_only_problems(base, current) @@ -255,24 +395,38 @@ def test_removing_or_reordering_entries_is_refused(specs, data): @settings(max_examples=150, deadline=None) @given( entry_specs.filter(bool), - st.sampled_from(registry.EXTENDING), + st.sampled_from(EXPECTED_EXTENDING), st.data(), ) def test_extending_lists_never_shrink(specs, key, data): base = build(specs) current = copy.deepcopy(base) - entry = data.draw(st.sampled_from(current["entries"])) - if not entry[key]: - entry[key] = ["something new"] - base_entry = base["entries"][current["entries"].index(entry)] - base_entry[key] = ["something old"] + index = data.draw(st.integers(0, len(specs) - 1)) + if not base["entries"][index][key]: + base["entries"][index][key] = ["something old"] + current["entries"][index][key] = ["something new"] else: - entry[key] = entry[key][1:] + current["entries"][index][key] = current["entries"][index][key][1:] assert registry.append_only_problems(base, current) -def test_rules_and_completeness_only_move_forward(): +def test_sources_rules_and_completeness_only_move_forward(): base = make_registry([]) + for change in ("remove", "edit"): + edited = copy.deepcopy(base) + if change == "remove": + del edited["sources"]["s_b"] + else: + edited["sources"]["s_b"]["family"] = "f2" + assert registry.append_only_problems(base, edited) + added = copy.deepcopy(base) + added["sources"]["s_d"] = { + "family": "f3", + "title": "D", + "catalog": "cat d", + "key_year": "y", + } + assert registry.append_only_problems(base, added) == [] trimmed = copy.deepcopy(base) trimmed["rules"] = trimmed["rules"][1:] assert registry.append_only_problems(base, trimmed) @@ -285,90 +439,171 @@ def test_rules_and_completeness_only_move_forward(): assert registry.append_only_problems(base, renamed) -def test_repeated_ids_are_refused(): - current = build([(["alpha"], "candidate"), (["beta:x"], "candidate")]) - current["entries"][1]["id"] = "H001" - assert any("ids repeat" in p for p in registry.validate(current)) +def test_an_alias_source_is_refused(): + current = make_registry([]) + current["sources"]["s_d"] = dict(SOURCES["s_a"], catalog=" CAT A ") + assert any("alias" in p for p in registry.validate(current)) + + +@given( + st.sampled_from(sorted(SOURCES)), + st.integers(1900, 2100), + st.integers(0, 5), + st.integers(1900, 2100), + st.integers(0, 5), + st.booleans(), +) +def test_keys_overlap_on_source_and_intersecting_years(s, y1, n1, y2, n2, x): + a = f"{s}:{y1}-{y1 + n1}:alpha" + b = f"{s}:{y2}-{y2 + n2}" + (":beta" if x else "") + intersect = y1 <= y2 + n2 and y2 <= y1 + n1 + assert registry.keys_overlap(a, b) == intersect + assert registry.keys_overlap(b, a) == intersect + assert registry.keys_overlap(a, f"{s}:undated") + other = "s_c" if s != "s_c" else "s_a" + assert not registry.keys_overlap(a, f"{other}:{y1}") + + +@pytest.mark.parametrize( + "key", + [ + "s_a", + "s_a:20", + "s_a:2022-2020", + "s_a:2020:Upper", + "S_A:2020", + "s_a:2020::x", + "s_a:dated", + 7, + ], +) +def test_malformed_unit_keys_are_refused(key): + entry = make_entry(0, ["s_a:2020"]) + entry["unit_keys"] = [key] + assert registry.validate(make_registry([entry])) + + +def test_a_key_naming_no_source_is_refused(): + entry = make_entry(0, ["s_z:2020"]) + problems = registry.validate(make_registry([entry])) + assert any("names no source" in p for p in problems) @pytest.mark.parametrize( - "statuses, last, message", + "provenance, statuses, last, message", [ - (["candidate", "used"], "used", "not a legal transition"), - (["used", "candidate"], "candidate", "not a legal transition"), - (["withdrawn", "frozen"], "frozen", "not a legal transition"), - (["candidate", "frozen"], "candidate", "is not its history's last"), + ("registration", ["frozen"], "frozen", "starts as one of"), + ("historical_import", ["candidate"], "candidate", "starts as one of"), + ("registration", ["candidate", "used"], "used", "not a legal"), + ("historical_import", ["used", "frozen"], "frozen", "not a legal"), + ("registration", ["candidate", "frozen"], "candidate", "last status"), ], ) -def test_status_follows_a_legal_history(statuses, last, message): - entry = make_entry(0, ["alpha"], "candidate") +def test_status_follows_a_legal_history(provenance, statuses, last, message): + entry = make_entry(0, ["s_a:2020"]) + entry["provenance"] = provenance entry["history"] = [ {"date": "2026-10-21", "status": s, "note": "n"} for s in statuses ] entry["status"] = last - entry["scored_evidence"] = ["x"] if last == "used" else [] + if last == "used": + entry["scored_evidence"] = [ + {"path": "s.json", "field": "gate", "equals": "gate_0"} + ] assert any(message in p for p in registry.validate(make_registry([entry]))) -def test_history_dates_never_go_back(): - entry = make_entry(0, ["alpha"], "frozen") - entry["history"][1]["date"] = "2026-10-01" - problems = registry.validate(make_registry([entry])) - assert any("goes back in time" in p for p in problems) +@pytest.mark.parametrize("date", ["2026-10-01", "2026-99-99", "yesterday"]) +def test_history_dates_are_real_and_never_go_back(date): + entry = make_entry(0, ["s_a:2020"], ("registration", "frozen")) + entry["history"][1]["date"] = date + assert registry.validate(make_registry([entry])) -def test_a_used_entry_needs_committed_scored_evidence(tmp_path): - entry = make_entry(0, ["alpha"], "used") - current = make_registry([entry]) - assert registry.validate(current) == [] - assert any( - "not a committed file" in p - for p in registry.validate(current, tmp_path) - ) - score = tmp_path / entry["scored_evidence"][0] - score.parent.mkdir(parents=True) - score.write_text("scored", encoding="utf-8") - assert registry.validate(current, tmp_path) == [] +def test_repeated_ids_and_shared_gate_identifiers_are_refused(): + current = build([(["s_a:2020"], ("registration", "candidate"))] * 2) + current["entries"][1]["id"] = "H001" + assert any("ids repeat" in p for p in registry.validate(current)) + current = build([(["s_a:2020"], ("registration", "candidate"))] * 2) + current["entries"][1]["gate_identifiers"].append("gate_0") + assert any("already identifies" in p for p in registry.validate(current)) + + +def test_scored_evidence_must_name_this_gate(): + entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + entry["scored_evidence"][0]["equals"] = "gate_other" + problems = registry.validate(make_registry([entry])) + assert any("not this gate" in p for p in problems) entry["scored_evidence"] = [] - assert any( - "needs scored_evidence" in p - for p in registry.validate(current, tmp_path) - ) - entry["scored_evidence"] = ["../outside.txt"] - assert any( - "not a committed file" in p - for p in registry.validate(current, tmp_path) - ) + problems = registry.validate(make_registry([entry])) + assert any("needs scored_evidence" in p for p in problems) -def test_complete_needs_its_completion_record(tmp_path): - current = make_registry([]) - current["complete"] = True - assert any( - "does not exist" in p for p in registry.validate(current, tmp_path) +def test_a_review_must_name_an_earlier_entry(): + current = build( + [(["s_a:2020"], ("registration", "candidate"))] + + [(["s_b:2025"], ("registration", "candidate"))] ) - record = tmp_path / current["completion_record"] - record.parent.mkdir(parents=True) - record.write_text("{}", encoding="utf-8") - assert registry.validate(current, tmp_path) == [] + first = current["entries"][0] + first["overlap_review"].append({"id": "H002", "reason": "later"}) + problems = registry.validate(current) + assert any("not an earlier entry" in p for p in problems) -@given(st.sampled_from(KEY_POOL), st.sampled_from(KEY_POOL)) -def test_key_overlap_is_symmetric_and_prefix_based(a, b): - assert registry.keys_overlap(a, a) - assert registry.keys_overlap(a, b) == registry.keys_overlap(b, a) - left, right = a.split(":"), b.split(":") - shorter = min(len(left), len(right)) - assert registry.keys_overlap(a, b) == (left[:shorter] == right[:shorter]) +@pytest.mark.parametrize( + "path, value", + [ + ((), None), + (("title",), None), + (("purpose",), []), + (("rules",), []), + (("status_values", "used"), 3), + (("sources", "s_a", "family"), ""), + (("complete",), "no"), + (("entries", 0, "scored_evidence"), 7), + (("entries", 0, "history"), "used"), + (("entries", 0, "history", 0), "used"), + (("entries", 0, "units"), {}), + (("entries", 0, "gate"), None), + (("entries", 0, "overlap_review"), [{"id": "H001"}]), + (("entries", 0, "disclosed_overlaps"), [3]), + ], +) +def test_malformed_fields_are_reported_not_raised(path, value): + current = build([(["s_a:2020"], ("historical_import", "used"))]) + if not path: + current = value + else: + target = current + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + problems = registry.validate(current) + assert isinstance(problems, list) and problems -def test_an_edition_overlaps_its_tables_but_not_its_neighbours(): - assert registry.keys_overlap("ssa_di_asr:2023", "ssa_di_asr:2023:table_19") - assert not registry.keys_overlap("ssa_di_asr:2023", "ssa_di_asr:2024") - assert not registry.keys_overlap("alpha:2021", "alpha:2021x") +json_values = st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=8), + lambda children: st.lists(children, max_size=4) + | st.dictionaries(st.text(max_size=8), children, max_size=4), + max_leaves=20, +) + + +@settings(max_examples=200, deadline=None) +@given(st.sampled_from(sorted(registry.ENTRY_KEYS) + ["entries"]), json_values) +def test_any_json_in_any_field_is_reported_not_raised(key, value): + base = build([(["s_a:2020"], ("historical_import", "used"))]) + current = copy.deepcopy(base) + if key == "entries": + current["entries"] = value + else: + current["entries"][0][key] = value + assert isinstance(registry.validate(current), list) + assert isinstance(registry.append_only_problems(base, current), list) -# The base registry, read through git -------------------------------------- +# Committed files and base commits, through git ---------------------------- def git(repository: Path, *args: str) -> str: @@ -393,32 +628,137 @@ def init_repository(tmp_path: Path) -> Path: return repository -def write_registry(repository: Path, current: dict) -> None: - path = repository / registry.REGISTRY +def write(repository: Path, relative: str, document) -> None: + path = repository / relative path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(current), encoding="utf-8") + text = document if isinstance(document, str) else json.dumps(document) + path.write_text(text, encoding="utf-8") + + +@pytest.fixture +def no_ci_env(monkeypatch): + for name in ("GITHUB_BASE_REF", "GITHUB_EVENT_NAME", "GITHUB_EVENT_PATH"): + monkeypatch.delenv(name, raising=False) + + +def test_scored_evidence_must_be_committed_and_identify_the_gate( + tmp_path, no_ci_env +): + repository = init_repository(tmp_path) + entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + current = make_registry([entry]) + evidence = entry["scored_evidence"][0] + write(repository, evidence["path"], {"gate": "gate_0"}) + assert any( + "not committed" in p for p in registry.validate(current, repository) + ) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "score") + assert registry.validate(current, repository) == [] + write(repository, "scores/other.json", {"gate": "gate_9"}) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "other") + evidence["path"] = "scores/other.json" + assert any( + "does not have gate" in p + for p in registry.validate(current, repository) + ) + write(repository, "scores/text.json", "not json") + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "text") + evidence["path"] = "scores/text.json" + assert any("not JSON" in p for p in registry.validate(current, repository)) + outside = tmp_path / "outside.json" + outside.write_text(json.dumps({"gate": "gate_0"}), encoding="utf-8") + (repository / "scores" / "link.json").symlink_to(outside) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "link") + evidence["path"] = "scores/link.json" + assert any("symlink" in p for p in registry.validate(current, repository)) + for path in ("../outside.json", "/etc/hosts", "~/x.json"): + evidence["path"] = path + assert any( + "outside the repository" in p + for p in registry.validate(current, repository) + ) + + +def test_complete_needs_its_completion_record(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = make_registry([], complete=True) + assert any( + "does not exist" in p for p in registry.validate(current, repository) + ) + write(repository, current["completion_record"], {}) + assert registry.validate(current, repository) == [] + + +def commit_registry(repository: Path, current: dict, message: str) -> str: + write(repository, registry.REGISTRY, current) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", message) + return git(repository, "rev-parse", "HEAD").strip() -def test_the_base_registry_is_read_at_the_merge_base(tmp_path, monkeypatch): - monkeypatch.delenv("GITHUB_BASE_REF", raising=False) +def test_a_pull_request_compares_with_its_merge_base(tmp_path, no_ci_env): repository = init_repository(tmp_path) git(repository, "checkout", "-q", "-b", "feature") + write(repository, "notes.txt", "a commit of the branch's own") + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "feature work") assert registry.base_registry(repository) is None git(repository, "checkout", "-q", "master") - first = build([(["alpha"], "candidate")]) - write_registry(repository, first) - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", "registry") + first = build([(["s_a:2020"], ("historical_import", "used"))]) + commit_registry(repository, first, "registry") git(repository, "checkout", "-q", "-b", "feature-2") second = copy.deepcopy(first) - second["entries"].append(make_entry(1, ["beta:x"], "candidate")) - write_registry(repository, second) - git(repository, "commit", "-q", "-am", "append") + second["entries"].append(make_entry(1, ["s_c:2099"])) + commit_registry(repository, second, "append") assert registry.base_registry(repository) == first -def test_no_base_branch_raises_rather_than_passing(tmp_path, monkeypatch): +def test_the_base_branch_compares_with_its_parent(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + first = build([(["s_a:2020"], ("historical_import", "used"))]) + commit_registry(repository, first, "registry") + emptied = copy.deepcopy(first) + emptied["entries"] = [] + commit_registry(repository, emptied, "delete H001") + base = registry.base_registry(repository) + assert base == first + assert registry.append_only_problems(base, emptied) + + +def test_a_push_compares_with_its_before_commit( + tmp_path, no_ci_env, monkeypatch +): + repository = init_repository(tmp_path) + first = build([(["s_a:2020"], ("historical_import", "used"))]) + before = commit_registry(repository, first, "registry") + middle = copy.deepcopy(first) + middle["entries"].append(make_entry(1, ["s_c:2099"])) + commit_registry(repository, middle, "append") + emptied = copy.deepcopy(first) + emptied["entries"] = [] + commit_registry(repository, emptied, "delete everything") + event = tmp_path / "event.json" + event.write_text(json.dumps({"before": before}), encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + assert registry.base_commit(repository) == before + assert registry.append_only_problems( + registry.base_registry(repository), emptied + ) + event.write_text(json.dumps({"before": "0" * 40}), encoding="utf-8") + assert registry.base_registry(repository) == middle + + +def test_no_base_raises_rather_than_passing(tmp_path, monkeypatch): + monkeypatch.delenv("GITHUB_EVENT_NAME", raising=False) monkeypatch.setenv("GITHUB_BASE_REF", "no-such-branch") repository = init_repository(tmp_path) with pytest.raises(LookupError): registry.base_registry(repository) + monkeypatch.delenv("GITHUB_BASE_REF") + with pytest.raises(LookupError): + registry.base_registry(repository) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index b41580fd1..058ab50fc 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 9983, + "unit": 10020, "artifact": 4253, "integration_psid": 1450, "reproduction_legacy": 520, From 9e3c6a4c7d5ff0824a8f53fa57ef73c3a86cba86 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 06:44:07 -0400 Subject: [PATCH 3/9] Holdout registry: answer review round 2 (six medium, one low) - EPUF's key year is the vintage in the file's title (the 2006 file, released in 2011); the seeded keys are pinned by an independently specified test. - The source vocabulary lives in the module (SOURCES), so adding a source or family is a reviewed code change; the registry's copy must equal it. Family reviews need the outcome "disjoint", a substantive reason (no placeholders, at least 40 characters) and path:line or URL citations checked against the committed tree; an unresolved relation is disclosed instead. The two deferred EPUF reviews are now disclosures. - Historical imports start as used, and their scored evidence must have been committed before the registry was. - Scored evidence must be a runs/*.json artifact read from HEAD's tree (not the working tree, not a symlink), matched on gate, gate.name or registration_id, whose value names the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. - On pushes, every pair of consecutive first-parent commits since the base must append only (history_problems), so a later commit cannot hide a change. - Appended entries with malformed histories are reported, not raised. - The family-review tests use an independently specified family map and overlap rule, with fixed same-family cases. - Tests split by tier: invented registries and temporary repositories (unit, 74) and the committed registry, which reads committed run artifacts (artifact, 8). 47 of 47 mutants killed. Tiers: unit 9,958 -> 10,032; artifact 4,253 -> 4,261. Co-Authored-By: Claude Opus 5.5 --- docs/design/track_b_holdout_registry.json | 45 +- .../track_b_holdout_registry_mutations.json | 267 ++++++--- .../track_b/holdout_registry.py | 518 ++++++++++++++---- tests/README-tiers.md | 6 +- tests/test_track_b_holdout_registry.py | 518 +++++++++++------- ...test_track_b_holdout_registry_committed.py | 155 ++++++ tests/tier_counts.json | 4 +- 7 files changed, 1110 insertions(+), 403 deletions(-) create mode 100644 tests/test_track_b_holdout_registry_committed.py diff --git a/docs/design/track_b_holdout_registry.json b/docs/design/track_b_holdout_registry.json index 03cd35555..a56381dcc 100644 --- a/docs/design/track_b_holdout_registry.json +++ b/docs/design/track_b_holdout_registry.json @@ -5,13 +5,13 @@ "rules": [ "A gate registers what it holds out here, as a lone candidate entry in a commit of its own, before its record freezes. It moves to frozen, used or withdrawn only in later commits.", "J's candidate holdouts are registered first, because J is on Track B's critical path. OP0.2 registers its picks for OP12's gate_ssa_totals here before it freezes them, and picks from the years J has not taken.", - "Every unit key names a declared source, then its years (a year, an inclusive range, or 'undated'), then optional scope segments. Each source declares a family, a title, a catalog reference no other source shares, and what its key year means. A new source is added only if it is not another name for a declared one.", - "Two unit keys share a unit when their source is the same and their years intersect; scope segments describe the unit and never narrow the overlap. Live entries that share a unit list each other in disclosed_overlaps, and both gates' registrations name the overlap. No unit is held out by two gates silently.", - "The overlap rule cannot see two sources that share underlying data, or two editions of a publication that report the same data years. So an entry lists in overlap_review, with a reason, every earlier live entry of the same source family that it neither shares a unit with nor discloses.", - "Reference years are recorded separately from interview or publication availability (design section 5.2, element 10): a unit key's year is what its source declares (for PSID, the interview wave), and each entry's units state the reference or event years.", - "An entry whose gate has scored has status 'used'. Its scored_evidence cites committed artifacts of that run, each with a field whose value identifies the gate. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", - "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused.", - "A historical import records a gate that froze or scored before this registry existed. Imports are allowed only while the registry is incomplete.", + "Every unit key names a source from the controlled vocabulary in src/populace_dynamics/track_b/holdout_registry.py (SOURCES), then its years (a year, an inclusive range, or 'undated'), then optional scope segments. The registry's sources must equal that vocabulary. Each source declares a family, a title, a catalog reference no other source shares, and what its key year means. Adding a source is a reviewed code change, and a new source must not be another name for a declared one.", + "Two unit keys share a unit when their source is the same and their years intersect; scope segments describe the unit and never narrow the overlap. Live entries that share a unit, or whose relation is unresolved, list each other in disclosed_overlaps, and both gates' registrations name the overlap. No unit is held out by two gates silently.", + "The overlap rule cannot see two sources that share underlying data, or two editions of a publication that report the same data years. So an entry reviews every earlier live entry of the same source family that it neither shares a unit with nor discloses. A review records the outcome 'disjoint', a substantive reason and citations (path:line or https URLs); an unresolved relation is disclosed instead.", + "Reference years are recorded separately from interview or publication availability (design section 5.2, element 10): a unit key's year is what its source declares (for PSID, the interview wave; for EPUF, the file's vintage year), and each entry's units state the reference or event years.", + "An entry whose gate has scored has status 'used'. Its scored_evidence cites committed scored-run artifacts under runs/, each with an identity field (gate, gate.name or registration_id) whose value identifies the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", + "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused. These rules hold against the base commit and between every pair of consecutive first-parent commits since it.", + "A historical import records a gate that scored before this registry existed: it starts as used, and its scored evidence must have been committed before the registry was. Imports are allowed only while the registry is incomplete.", "The registry is incomplete until completion_record, B0.2's mechanical exposure inventory, exists. It starts with the prior uses verified below; the inventory adds the rest before any B0.2 gate record freezes." ], "status_values": { @@ -31,7 +31,7 @@ "family": "ssa_administrative", "title": "SSA Earnings Public-Use File", "catalog": "SSA Office of Research, Evaluation, and Statistics: Earnings Public-Use File", - "key_year": "file release year" + "key_year": "the vintage year in the file's title: the 2006 file holds earnings through 2006 and was released in August 2011" }, "ssa_supplement": { "family": "ssa_administrative", @@ -47,7 +47,7 @@ }, "census_cps_tables": { "family": "us_cross_section", - "title": "U.S. Census Bureau Current Population Survey published tables", + "title": "U.S. Census Bureau Current Population Survey tables", "catalog": "U.S. Census Bureau: Current Population Survey published tables", "key_year": "table reference year" }, @@ -223,13 +223,16 @@ "equals": "2026-10-03-epuf-career-fill" } ], - "disclosed_overlaps": [], + "disclosed_overlaps": [ + "H004", + "H005" + ], "overlap_review": [], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; open_population.md rev 2 plans gate_young_earn on a new salt of the same file, which shares this unit and must disclose it" + "note": "seeded from gates.yaml at 1e739d32; its relation to the DI ASR 2023 (H004, H005) is unresolved and disclosed; open_population.md rev 2 plans gate_young_earn on a new salt of the same file, which shares this unit and must disclose it" } ] }, @@ -277,19 +280,15 @@ "disclosed_overlaps": [ "H001", "H002", + "H003", "H005" ], - "overlap_review": [ - { - "id": "H003", - "reason": "EPUF 2006 is a person-level extract of earnings through 2006 (src/populace_dynamics/data/epuf.py:15); gate_m4 holds out tables of the DI ASR's 2023 edition. They share no edition or file year. Whether a DI ASR 2023 table reports data years before 2007 is for B0.2's inventory to itemize." - } - ], + "overlap_review": [], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; its PSID panel shares waves with gate_m6 (H001) and B2 (H002), and the DI ASR 2023 edition with gate_w1 (H005)" + "note": "seeded from gates.yaml at 1e739d32; its PSID panel shares waves with gate_m6 (H001) and B2 (H002), and the DI ASR 2023 edition with gate_w1 (H005); whether a DI ASR 2023 table reports data years that EPUF 2006 also covers (H003) is unresolved, so it is disclosed" } ] }, @@ -357,19 +356,15 @@ } ], "disclosed_overlaps": [ + "H003", "H004" ], - "overlap_review": [ - { - "id": "H003", - "reason": "EPUF 2006 is a person-level extract of earnings through 2006 (src/populace_dynamics/data/epuf.py:15); gate_w1 used tables of the Supplement's and the DI ASR's 2023 editions. They share no edition or file year. The data years those tables report are for B0.2's inventory to itemize." - } - ], + "overlap_review": [], "history": [ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; shares the DI ASR 2023 edition with gate_m4 (H004)" + "note": "seeded from gates.yaml at 1e739d32; shares the DI ASR 2023 edition with gate_m4 (H004); whether its 2023 Supplement and DI ASR tables report data years that EPUF 2006 also covers (H003) is unresolved, so it is disclosed" } ] } diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json index e2d749c8c..199fa932c 100644 --- a/docs/design/track_b_holdout_registry_mutations.json +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -2,16 +2,16 @@ "schema_version": 1, "verification_class": "accounting conformance", "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", - "mutation_method": "One deliberate production-line change per check; the full registry test module runs and is expected to fail; original bytes restored before the next check. No mutated implementation is committed.", + "mutation_method": "One deliberate production-line change per check; both registry test modules run in full and are expected to fail; original bytes restored before the next check. No mutated implementation is committed.", "restored_source_sha256": { - "src/populace_dynamics/track_b/holdout_registry.py": "599e8966fd75801ee1f3f45af3eb4a8aa06b4d2c017cec898e7d58cb19f55f5b" + "src/populace_dynamics/track_b/holdout_registry.py": "b1064e0fe899556110f8eccb9c4cf914d2dd2fdee8c455c0b49d88cd73b2026c" }, "checks": [ { "mutation": "scope segments unchecked", "old": "if not all(SCOPE_RX.fullmatch(part) for part in parts[2:]):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2020::x]", @@ -22,7 +22,7 @@ "mutation": "reversed year range accepted", "old": "if end < start:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_malformed_unit_keys_are_refused[s_a:2022-2020]" @@ -32,91 +32,190 @@ "mutation": "different sources overlap", "old": "if left is None or right is None or left[0] != right[0]:", "new": "if left is None or right is None:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen", + "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused", + "tests/test_track_b_holdout_registry.py::test_appending_lone_candidates_keeps_every_rule", "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", - "tests/test_track_b_holdout_registry.py::test_the_committed_registry_keeps_its_rules" + "tests/test_track_b_holdout_registry.py::test_settled_registries_are_valid", + "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", + "tests/test_track_b_holdout_registry_committed.py::test_the_committed_registry_keeps_its_rules" ] }, { "mutation": "undated overlaps nothing", "old": "if left[1] is None or right[1] is None:\n return True", "new": "if left[1] is None or right[1] is None:\n return False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years" + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", + "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", + "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule" ] }, { "mutation": "ranges overlap only when equal", "old": "return left[1][0] <= right[1][1] and right[1][0] <= left[1][1]", "new": "return left[1] == right[1]", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", - "tests/test_track_b_holdout_registry.py::test_the_seeded_shared_units_are_disclosed_both_ways[H001-H002]", - "tests/test_track_b_holdout_registry.py::test_the_seeded_shared_units_are_disclosed_both_ways[H001-H004]", - "tests/test_track_b_holdout_registry.py::test_the_seeded_shared_units_are_disclosed_both_ways[H002-H004]" + "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", + "tests/test_track_b_holdout_registry_committed.py::test_the_seeded_disclosures_are_as_specified" ] }, { - "mutation": "evidence may name another gate", - "old": "if equals not in identifiers:", + "mutation": "symlinks read as committed files", + "old": "and fields[0] in (\"100644\", \"100755\")", + "new": "and fields[0] in (\"100644\", \"100755\", \"120000\")", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused" + ] + }, + { + "mutation": "unknown gates accepted", + "old": "if not authoritative:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_name_this_gate" + "tests/test_track_b_holdout_registry.py::test_identifiers_must_be_authoritative" ] }, { - "mutation": "evidence paths may leave the repository", - "old": "if path.startswith((\"/\", \"~\")) or \"..\" in path.split(\"/\"):", + "mutation": "self-declared identifiers accepted", + "old": "allowed &= authoritative", + "new": "pass", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_identifiers_must_be_authoritative" + ] + }, + { + "mutation": "evidence outside runs/ accepted", + "old": "if not RUN_PATH_RX.fullmatch(path):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-is not runs/]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-is not runs/]" ] }, { - "mutation": "evidence need not be committed", - "old": "if path not in tracked:", + "mutation": "any field may identify", + "old": "if field not in IDENTITY_FIELDS:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change1-not an identity field]" ] }, { - "mutation": "evidence may be a symlink", - "old": "if target.is_symlink() or not resolved.is_relative_to(root):", + "mutation": "evidence may name another gate", + "old": "if equals not in allowed:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + "tests/test_track_b_holdout_registry.py::test_identifiers_must_be_authoritative", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change0-does not identify this gate]" + ] + }, + { + "mutation": "evidence need not be committed", + "old": "if text is None:\n problems.append(\n f\"{name}: scored evidence {path!r} is not a committed file\"", + "new": "if False:\n problems.append(\n f\"{name}: scored evidence {path!r} is not a committed file\"", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_is_read_from_the_committed_tree" ] }, { "mutation": "evidence need not identify the gate", "old": "if lookup(document, field) != equals:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_is_read_from_the_committed_tree" + ] + }, + { + "mutation": "late imports accepted", + "old": "if added is None or not committed.is_ancestor(added, cutoff):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_need_evidence_older_than_the_registry" + ] + }, + { + "mutation": "any review outcome", + "old": "if item[\"outcome\"] not in REVIEW_OUTCOMES:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_committed_and_identify_the_gate" + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change0-is disclosed]" + ] + }, + { + "mutation": "placeholder reasons accepted", + "old": "if len(reason) < MIN_REASON or PLACEHOLDER_RX.search(reason):", + "new": "if len(reason) < MIN_REASON:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change1-substantive]" + ] + }, + { + "mutation": "reviews need no citations", + "old": "if not isinstance(citations, list) or not citations:", + "new": "if not isinstance(citations, list):", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change3-cites nothing]" + ] + }, + { + "mutation": "citations need no form", + "old": "if match is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change4-not path:line or a URL]" + ] + }, + { + "mutation": "citation lines unbounded", + "old": "if end > len(text.splitlines()):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_review_citations_must_name_committed_lines" ] }, { "mutation": "keys may name undeclared sources", "old": "elif parsed[0] not in sources:", "new": "elif False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_key_naming_no_source_is_refused" @@ -126,10 +225,11 @@ "mutation": "any first status", "old": "if history[0][\"status\"] not in first:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses1-candidate-starts as one of]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses2-frozen-starts as one of]", "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses0-frozen-starts as one of]" ] }, @@ -137,7 +237,7 @@ "mutation": "dates not parsed", "old": "date = dt.date.fromisoformat(item[\"date\"])", "new": "date = previous_date", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-10-01]", @@ -149,7 +249,7 @@ "mutation": "dates may go back", "old": "if previous_date is not None and date < previous_date:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_history_dates_are_real_and_never_go_back[2026-10-01]" @@ -159,58 +259,72 @@ "mutation": "any transition", "old": "status not in TRANSITIONS[previous_status]", "new": "False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses3-frozen-not a legal]", - "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses2-used-not a legal]" + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses4-frozen-not a legal]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses3-used-not a legal]" ] }, { "mutation": "status need not match history", "old": "if entry[\"status\"] != previous_status:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses4-candidate-last status]" + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses5-candidate-last status]" ] }, { "mutation": "used needs no scored evidence", "old": "if entry[\"status\"] == \"used\" and not entry[\"scored_evidence\"]:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_name_this_gate" + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change0-does not identify this gate]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change1-not an identity field]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-is not runs/]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-is not runs/]" ] }, { "mutation": "alias sources accepted", "old": "if catalog in catalogs:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_an_alias_source_is_refused" + "tests/test_track_b_holdout_registry.py::test_the_registry_must_carry_the_controlled_vocabulary" + ] + }, + { + "mutation": "registry may carry its own vocabulary", + "old": "if registry[\"sources\"] != dict(vocabulary):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_fields_are_reported_not_raised[path5-]", + "tests/test_track_b_holdout_registry.py::test_the_registry_must_carry_the_controlled_vocabulary" ] }, { "mutation": "complete needs no record", - "old": "and registry[\"complete\"]", + "old": "and committed.text(record) is None", "new": "and False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_complete_needs_its_completion_record" + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_committed_record" ] }, { "mutation": "one identifier, two gates", "old": "if owner != entry[\"gate\"]:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_repeated_ids_and_shared_gate_identifiers_are_refused" @@ -220,7 +334,7 @@ "mutation": "one-way disclosure accepted", "old": "elif entry[\"id\"] not in by_id[other][\"disclosed_overlaps\"]:", "new": "elif False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_one_way_disclosure_is_refused" @@ -230,7 +344,7 @@ "mutation": "withdrawn entries counted live", "old": "live = [entry for entry in entries if entry[\"status\"] != \"withdrawn\"]", "new": "live = list(entries)", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen", @@ -243,7 +357,7 @@ "mutation": "silent shared unit accepted", "old": "other[\"id\"] not in entry[\"disclosed_overlaps\"]\n ):", "new": "False\n ):", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused" @@ -253,7 +367,7 @@ "mutation": "reviews may name later entries", "old": "if position.get(name, index) >= index:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_review_must_name_an_earlier_entry" @@ -263,17 +377,18 @@ "mutation": "family relations need no review", "old": "and other[\"id\"] not in reviewed", "new": "and False", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused" + "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused", + "tests/test_track_b_holdout_registry.py::test_fixed_family_cases_follow_the_rule" ] }, { "mutation": "rules may shrink", "old": "if not _extends(base.get(\"rules\", []), current.get(\"rules\", [])):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" @@ -283,7 +398,7 @@ "mutation": "sources may change", "old": "if sources.get(source) != declared:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" @@ -293,7 +408,7 @@ "mutation": "complete may revert", "old": "if base.get(\"complete\") is True and current.get(\"complete\") is not True:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_sources_rules_and_completeness_only_move_forward" @@ -303,11 +418,11 @@ "mutation": "entries may be removed or reordered", "old": "if new_ids[: len(old_ids)] != old_ids:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_cannot_hide_a_change_a_later_commit_undoes[append_then_remove]", "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", - "tests/test_track_b_holdout_registry.py::test_any_json_in_any_field_is_reported_not_raised", "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused", "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" ] @@ -316,9 +431,10 @@ "mutation": "immutable fields may change", "old": "if after.get(key) != before.get(key):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_cannot_hide_a_change_a_later_commit_undoes[edit_then_restore]", "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused" ] }, @@ -326,7 +442,7 @@ "mutation": "extending lists may shrink", "old": "if not _extends(before.get(key, []), after.get(key, [])):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_extending_lists_never_shrink" @@ -336,7 +452,7 @@ "mutation": "imports allowed after completion", "old": "if base.get(\"complete\") is True:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_imports_close_when_the_registry_is_complete" @@ -344,9 +460,9 @@ }, { "mutation": "registrations may arrive frozen", - "old": "elif [item.get(\"status\") for item in entry.get(\"history\", [])] != [", - "new": "elif False and [item.get(\"status\") for item in entry.get(\"history\", [])] != [", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "old": "elif statuses != [\"candidate\"]:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen" @@ -356,7 +472,7 @@ "mutation": "push ignores its before commit", "old": "if set(before) != {\"0\"} and _commit(repository, before):", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit" @@ -366,7 +482,7 @@ "mutation": "base branch compares with itself", "old": "if base == head:", "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", @@ -376,12 +492,23 @@ }, { "mutation": "absent base registry raises", - "old": "if not listed.stdout.strip():", - "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py", + "old": "if not listed.stdout.strip():\n return None", + "new": "if False:\n return None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_pull_request_compares_with_its_merge_base", + "tests/test_track_b_holdout_registry_committed.py::test_the_committed_registry_only_appends_to_its_base" + ] + }, + { + "mutation": "history checks only the endpoint", + "old": "previous_commit, previous = commit, current", + "new": "previous_commit = commit", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_pull_request_compares_with_its_merge_base" + "tests/test_track_b_holdout_registry.py::test_a_push_cannot_hide_a_change_a_later_commit_undoes[append_then_remove]" ] } ] diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py index 903885ca4..0645033e1 100644 --- a/src/populace_dynamics/track_b/holdout_registry.py +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -6,29 +6,33 @@ This module enforces its rules: - one schema, ``track_b_holdout_registry.v1``, with every field typed; -- a declared source vocabulary: each source has a family, a title and a - catalog reference that no other source shares, so one source cannot be - registered under two names; +- a source vocabulary held here, in :data:`SOURCES`, so that adding a + source or a family is a reviewed code change. Each source has a family, + a title, a catalog reference no other source shares, and what its key + year means. The registry's copy must equal it; - unit keys ``source:years[:scope...]``, where ``years`` is a year, an inclusive year range, or ``undated``. Two keys name a shared unit when they have the same source and their years intersect (``undated`` intersects every year). Scope segments describe the unit and never narrow the overlap; -- no shared unit between live entries goes undisclosed, and every - disclosure is symmetric; -- relations the mechanical rule cannot see are reviewed by hand: an entry - lists, with a reason, every earlier live entry of the same source - family that it neither shares a unit with nor discloses; +- live entries that share a unit, or whose relation is unresolved, list + each other in ``disclosed_overlaps``, and every disclosure is symmetric; +- relations the mechanical rule cannot see are reviewed: an entry lists + every earlier live entry of the same source family that it neither + shares a unit with nor discloses, with the outcome ``disjoint``, a + substantive reason and citations. An unresolved relation is disclosed + instead; - unique entry ids, which are never removed, reordered or reused; -- histories with legal transitions only. A registration starts as a - ``candidate``, and is added in a commit of its own before it may move - on, so every registration reaches the base branch before its gate - freezes. A historical import, a gate that froze or scored before the - registry existed, starts as ``frozen`` or ``used`` and may be added only - while the registry is incomplete; -- a ``used`` entry cites committed, gate-identified artifacts of its - scored run; -- entries only grow: against the base registry, an entry's identity and +- histories with legal transitions only. A registration starts as a lone + ``candidate`` in a commit of its own, so it reaches the base branch + before its gate freezes. A historical import is a gate that scored + before the registry existed: it starts as ``used``, and its scored + evidence must have been committed before the registry was; +- a ``used`` entry cites scored-run artifacts under ``runs/``, read from + the committed tree, whose identity field equals one of the gate's + identifiers in ``gates.yaml`` or :data:`REGISTRATIONS_OUTSIDE_GATES_YAML`; +- entries only grow: against the base registry, and between every pair + of consecutive first-parent commits since it, an entry's identity and units never change, and its lists only extend; - the registry stays incomplete until its completion record, B0.2's mechanical exposure inventory, exists. @@ -37,7 +41,8 @@ the same source at the same year, whatever variable or table they use. It cannot see that two sources share underlying data, or that two editions of a publication report the same data years. That is what the -family review is for. +family review and disclosures are for, and what reviewers of each +registry change must check. Each check returns a list of problems; an empty list means the registry keeps the rules. @@ -50,12 +55,106 @@ import os import re import subprocess -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType from typing import Any +import yaml + SCHEMA = "track_b_holdout_registry.v1" REGISTRY = "docs/design/track_b_holdout_registry.json" +GATES = "gates.yaml" + +#: The source vocabulary. Adding a source is a reviewed code change, and a +#: new id must not be another name for a declared source. +SOURCES: Mapping[str, Mapping[str, str]] = MappingProxyType( + { + "psid": { + "family": "psid", + "title": ( + "Panel Study of Income Dynamics: family files, cross-year " + "individual file and supplements" + ), + "catalog": ( + "University of Michigan Institute for Social Research: " + "Panel Study of Income Dynamics" + ), + "key_year": ( + "interview wave (collection year); income reference years " + "are the wave year minus one" + ), + }, + "ssa_epuf": { + "family": "ssa_administrative", + "title": "SSA Earnings Public-Use File", + "catalog": ( + "SSA Office of Research, Evaluation, and Statistics: " + "Earnings Public-Use File" + ), + "key_year": ( + "the vintage year in the file's title: the 2006 file holds " + "earnings through 2006 and was released in August 2011" + ), + }, + "ssa_supplement": { + "family": "ssa_administrative", + "title": ( + "Annual Statistical Supplement to the Social Security " + "Bulletin" + ), + "catalog": ( + "SSA Office of Research, Evaluation, and Statistics: Annual " + "Statistical Supplement to the Social Security Bulletin" + ), + "key_year": "edition year", + }, + "ssa_di_asr": { + "family": "ssa_administrative", + "title": ( + "Annual Statistical Report on the Social Security " + "Disability Insurance Program" + ), + "catalog": ( + "SSA Office of Research, Evaluation, and Statistics: Annual " + "Statistical Report on the Social Security Disability " + "Insurance Program" + ), + "key_year": "edition year", + }, + "census_cps_tables": { + "family": "us_cross_section", + "title": "U.S. Census Bureau Current Population Survey tables", + "catalog": ( + "U.S. Census Bureau: Current Population Survey published " + "tables" + ), + "key_year": "table reference year", + }, + "populace_us_frame": { + "family": "us_cross_section", + "title": "Microcosm US certified frame", + "catalog": "hf://policyengine/populace-us/populace_us_2024.h5", + "key_year": "frame vintage", + }, + "mermin_smith_anchors": { + "family": "literature", + "title": ( + "Mermin and Smith published orderings, committed as anchors" + ), + "catalog": ( + "gates.yaml gate_w1 holdout_basis: " + "mermin_smith_committed_orderings" + ), + "key_year": "undated", + }, + } +) +#: Gates registered outside ``gates.yaml`` (Track B's #42 registrations), +#: with every identifier that may name them in scored evidence. +REGISTRATIONS_OUTSIDE_GATES_YAML: Mapping[str, tuple[str, ...]] = ( + MappingProxyType({"B2": ("B2", "6252388925")}) +) STATUSES = ("candidate", "frozen", "used", "withdrawn") TRANSITIONS = { "candidate": frozenset({"frozen", "withdrawn"}), @@ -63,11 +162,10 @@ "used": frozenset(), "withdrawn": frozenset(), } -#: Each provenance's permitted first statuses. A historical import records -#: a gate that froze or scored before the registry existed. +#: Each provenance's permitted first statuses. FIRST_STATUS = { "registration": frozenset({"candidate"}), - "historical_import": frozenset({"frozen", "used"}), + "historical_import": frozenset({"used"}), } TEXT_FIELDS = ("schema", "title", "purpose", "completion_record") TOP_LEVEL = ( @@ -109,10 +207,30 @@ "overlap_review", "history", ) +ITEM_FIELDS = { + "scored_evidence": ("path", "field", "equals"), + "overlap_review": ("id", "outcome", "reason"), + "history": ("date", "status", "note"), +} +#: Identity fields a scored-run artifact may be matched on. +IDENTITY_FIELDS = ("gate", "gate.name", "registration_id") +REVIEW_OUTCOMES = ("disjoint",) +MIN_REASON = 40 +PLACEHOLDER_RX = re.compile( + r"\b(todo|tbd|fixme|xxx|placeholder|to be determined)\b", re.I +) ID_RX = re.compile(r"H[0-9]{3,}") SOURCE_RX = re.compile(r"[a-z][a-z0-9_]*") YEARS_RX = re.compile(r"(?P[0-9]{4})(?:-(?P[0-9]{4}))?|undated") SCOPE_RX = re.compile(r"[a-z0-9][a-z0-9_.\-]*") +#: Scored-run artifacts live here. +RUNS_DIR = "runs" +RUN_PATH_RX = re.compile(rf"{RUNS_DIR}/[A-Za-z0-9_.\-]+\.json") +CITATION_RX = re.compile( + r"(?P[A-Za-z0-9_][A-Za-z0-9_./\-]*):(?P[0-9]+)" + r"(?:-(?P[0-9]+))?" +) +URL_RX = re.compile(r"https://\S+") SHA_RX = re.compile(r"[0-9a-f]{40}") @@ -174,6 +292,9 @@ def families(entry: Mapping[str, Any], sources: Mapping[str, Any]) -> set[str]: } +_MISSING = object() + + def lookup(document: Any, dotted: str) -> Any: """``document``'s value at a dotted field path, or a sentinel.""" for part in dotted.split("."): @@ -183,55 +304,202 @@ def lookup(document: Any, dotted: str) -> Any: return document -_MISSING = object() +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", str(repository), *args], + capture_output=True, + text=True, + check=False, + ) -def tracked_files(repository: Path) -> set[str] | None: - """Paths git tracks in ``repository``, or ``None`` without git.""" - listed = _git(repository, "ls-files", "-z") - if listed.returncode != 0: - return None - return {path for path in listed.stdout.split("\0") if path} +class Committed: + """Read-only access to one commit of ``repository``, cached.""" + + def __init__(self, repository: Path, commit: str = "HEAD") -> None: + self.repository = repository + self.commit = commit + self._files: dict[str, str | None] = {} + self._gates: Mapping[str, Any] | None = None + self._first_added: dict[str, str | None] = {} + + def text(self, path: str) -> str | None: + """A regular committed file's text, or ``None``.""" + if path not in self._files: + listed = _git(self.repository, "ls-tree", self.commit, "--", path) + fields = listed.stdout.split() + regular = ( + listed.returncode == 0 + and len(fields) >= 4 + and fields[0] in ("100644", "100755") + and fields[1] == "blob" + ) + shown = ( + _git(self.repository, "show", f"{self.commit}:{path}") + if regular + else None + ) + self._files[path] = ( + shown.stdout if shown and shown.returncode == 0 else None + ) + return self._files[path] + + def identifiers(self, gate: str) -> set[str]: + """Every identifier that may name ``gate`` in scored evidence.""" + if self._gates is None: + text = self.text(GATES) + document = yaml.safe_load(text) if text else {} + gates = ( + document.get("gates") if isinstance(document, dict) else None + ) + self._gates = gates if isinstance(gates, Mapping) else {} + names = set(REGISTRATIONS_OUTSIDE_GATES_YAML.get(gate, ())) + declared = self._gates.get(gate) + if isinstance(declared, Mapping): + names.add(gate) + for key in ("id", "registration_id"): + if isinstance(declared.get(key), str): + names.add(declared[key]) + return names + + def first_added(self, path: str) -> str | None: + """The oldest commit before ``self.commit`` that added ``path``.""" + if path not in self._first_added: + log = _git( + self.repository, + "log", + "--diff-filter=A", + "--format=%H", + self.commit, + "--", + path, + ) + commits = log.stdout.split() if log.returncode == 0 else [] + self._first_added[path] = commits[-1] if commits else None + return self._first_added[path] + + def is_ancestor(self, older: str, newer: str) -> bool: + return ( + _git( + self.repository, "merge-base", "--is-ancestor", older, newer + ).returncode + == 0 + ) def _evidence_problems( - name: str, - item: Mapping[str, Any], - identifiers: Sequence[str], - repository: Path | None, - tracked: set[str] | None, + entry: Mapping[str, Any], committed: Committed | None ) -> list[str]: - """Problems with one scored-evidence citation.""" - path, field, equals = item["path"], item["field"], item["equals"] - if equals not in identifiers: - return [f"{name}: scored evidence names {equals!r}, not this gate"] - if repository is None: + """Problems with an entry's scored-evidence citations.""" + if not entry["scored_evidence"]: return [] - if path.startswith(("/", "~")) or ".." in path.split("/"): - return [f"{name}: scored evidence {path!r} is outside the repository"] - if tracked is None: - return [f"{name}: cannot list committed files to check {path!r}"] - if path not in tracked: - return [f"{name}: scored evidence {path!r} is not committed"] - target = repository / path - root = repository.resolve() - resolved = target.resolve() - if target.is_symlink() or not resolved.is_relative_to(root): - return [ - f"{name}: scored evidence {path!r} is a symlink or leaves " - "the repository" - ] - if not resolved.is_file(): - return [f"{name}: scored evidence {path!r} is not a file"] - try: - document = json.loads(resolved.read_text(encoding="utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - return [f"{name}: scored evidence {path!r} is not JSON"] - if lookup(document, field) != equals: - return [ - f"{name}: scored evidence {path!r} does not have " - f"{field} == {equals!r}" - ] + name = entry["id"] + problems: list[str] = [] + allowed = set(entry["gate_identifiers"]) + if committed is not None: + authoritative = committed.identifiers(entry["gate"]) + if not authoritative: + return [ + f"{name}: {entry['gate']!r} is neither a gates.yaml gate nor " + "a registration this module lists" + ] + allowed &= authoritative + cutoff = None + if committed is not None and entry["provenance"] == "historical_import": + cutoff = committed.first_added(REGISTRY) + for item in entry["scored_evidence"]: + path, field, equals = item["path"], item["field"], item["equals"] + if not RUN_PATH_RX.fullmatch(path): + problems.append( + f"{name}: scored evidence {path!r} is not runs/*.json" + ) + continue + if field not in IDENTITY_FIELDS: + problems.append(f"{name}: {field!r} is not an identity field") + continue + if equals not in allowed: + problems.append( + f"{name}: scored evidence names {equals!r}, which does not " + "identify this gate" + ) + continue + if committed is None: + continue + text = committed.text(path) + if text is None: + problems.append( + f"{name}: scored evidence {path!r} is not a committed file" + ) + continue + try: + document = json.loads(text) + except json.JSONDecodeError: + problems.append(f"{name}: scored evidence {path!r} is not JSON") + continue + if lookup(document, field) != equals: + problems.append( + f"{name}: scored evidence {path!r} does not have " + f"{field} == {equals!r}" + ) + continue + if cutoff is not None: + added = committed.first_added(path) + if added is None or not committed.is_ancestor(added, cutoff): + problems.append( + f"{name}: a historical import's evidence {path!r} was " + "committed after the registry existed" + ) + return problems + + +def _review_problems( + entry: Mapping[str, Any], committed: Committed | None +) -> list[str]: + """Problems with an entry's family-review items.""" + name = entry["id"] + problems: list[str] = [] + for item in entry["overlap_review"]: + reviewed = item["id"] + if item["outcome"] not in REVIEW_OUTCOMES: + problems.append( + f"{name}: review of {reviewed} has outcome " + f"{item['outcome']!r}; an unresolved relation is disclosed" + ) + reason = item["reason"].strip() + if len(reason) < MIN_REASON or PLACEHOLDER_RX.search(reason): + problems.append( + f"{name}: review of {reviewed} needs a substantive reason" + ) + citations = item.get("evidence") + if not isinstance(citations, list) or not citations: + problems.append(f"{name}: review of {reviewed} cites nothing") + continue + for citation in citations: + problems.extend(_citation_problems(name, citation, committed)) + return problems + + +def _citation_problems( + name: str, citation: Any, committed: Committed | None +) -> list[str]: + if not isinstance(citation, str): + return [f"{name}: a review citation is not a string"] + if URL_RX.fullmatch(citation): + return [] + match = CITATION_RX.fullmatch(citation) + if match is None: + return [f"{name}: citation {citation!r} is not path:line or a URL"] + start = int(match["start"]) + end = int(match["end"] or start) + if not 1 <= start <= end: + return [f"{name}: citation {citation!r} has a bad line range"] + if committed is None: + return [] + text = committed.text(match["path"]) + if text is None: + return [f"{name}: citation {citation!r} names no committed file"] + if end > len(text.splitlines()): + return [f"{name}: citation {citation!r} runs past the file's end"] return [] @@ -257,20 +525,11 @@ def _shape_problems(entry: Any) -> list[str]: problems.append(f"{name}: units must be a non-empty mapping") if problems: return problems - for key in ( - "gate_identifiers", - "unit_keys", - "evidence", - "disclosed_overlaps", - ): + plain = ("gate_identifiers", "unit_keys", "evidence", "disclosed_overlaps") + for key in plain: if not all(_text(item) for item in entry[key]): problems.append(f"{name}: {key} must hold non-empty strings") - shapes = { - "scored_evidence": ("path", "field", "equals"), - "overlap_review": ("id", "reason"), - "history": ("date", "status", "note"), - } - for key, fields in shapes.items(): + for key, fields in ITEM_FIELDS.items(): for item in entry[key]: if not isinstance(item, Mapping) or not all( _text(item.get(field)) for field in fields @@ -286,8 +545,7 @@ def _shape_problems(entry: Any) -> list[str]: def _entry_problems( entry: Mapping[str, Any], sources: Mapping[str, Any], - repository: Path | None, - tracked: set[str] | None, + committed: Committed | None, ) -> list[str]: """Problems inside one well-shaped entry.""" name = entry["id"] @@ -347,12 +605,8 @@ def _entry_problems( ) if entry["status"] == "used" and not entry["scored_evidence"]: problems.append(f"{name}: a used entry needs scored_evidence") - for item in entry["scored_evidence"]: - problems.extend( - _evidence_problems( - name, item, entry["gate_identifiers"], repository, tracked - ) - ) + problems.extend(_evidence_problems(entry, committed)) + problems.extend(_review_problems(entry, committed)) return problems @@ -381,10 +635,15 @@ def _vocabulary_problems(sources: Any) -> list[str]: return problems -def validate(registry: Any, repository: Path | None = None) -> list[str]: - """Every rule one registry must keep on its own. With ``repository``, - scored evidence and the completion record are checked against its - committed files.""" +def validate( + registry: Any, + repository: Path | None = None, + vocabulary: Mapping[str, Mapping[str, str]] = SOURCES, +) -> list[str]: + """Every rule one registry must keep on its own. ``vocabulary`` is the + controlled source list the registry's copy must equal. With + ``repository``, scored evidence, citations and the completion record + are checked against its ``HEAD`` commit.""" if not isinstance(registry, Mapping): return ["registry: not a mapping"] missing = [key for key in TOP_LEVEL if key not in registry] @@ -413,7 +672,11 @@ def validate(registry: Any, repository: Path | None = None) -> list[str]: problems.append("registry: status_values must describe each status") if not isinstance(registry["complete"], bool): problems.append("registry: complete must be true or false") - problems.extend(_vocabulary_problems(registry["sources"])) + problems.extend(_vocabulary_problems(vocabulary)) + if registry["sources"] != dict(vocabulary): + problems.append( + "registry: sources must equal the controlled vocabulary" + ) entries = registry["entries"] if not isinstance(entries, list): problems.append("registry: entries must be a list") @@ -423,19 +686,18 @@ def validate(registry: Any, repository: Path | None = None) -> list[str]: problems.extend(_shape_problems(entry)) if problems: return problems + committed = Committed(repository) if repository is not None else None record = registry["completion_record"] if ( - repository is not None + committed is not None and registry["complete"] - and not (repository / record).is_file() + and committed.text(record) is None ): problems.append( - f"registry: marked complete, but {record} does not exist" + f"registry: marked complete, but {record} is not committed" ) - sources = registry["sources"] - tracked = tracked_files(repository) if repository is not None else None for entry in entries: - problems.extend(_entry_problems(entry, sources, repository, tracked)) + problems.extend(_entry_problems(entry, vocabulary, committed)) if problems: return problems ids = [entry["id"] for entry in entries] @@ -483,11 +745,11 @@ def validate(registry: Any, repository: Path | None = None) -> list[str]: ) if entry["status"] == "withdrawn": continue - mine = families(entry, sources) + mine = families(entry, vocabulary) for other in entries[:index]: if ( other["status"] != "withdrawn" - and mine & families(other, sources) + and mine & families(other, vocabulary) and not entries_overlap(entry, other) and other["id"] not in entry["disclosed_overlaps"] and other["id"] not in reviewed @@ -549,15 +811,20 @@ def append_only_problems( problems.append(f"{before['id']}: {key} may only extend") for entry in new[len(old) :]: name = entry.get("id") + history = entry.get("history") + statuses = ( + [item.get("status") for item in history] + if isinstance(history, list) + and all(isinstance(item, Mapping) for item in history) + else None + ) if entry.get("provenance") == "historical_import": if base.get("complete") is True: problems.append( f"{name}: historical imports close when the registry " "is complete" ) - elif [item.get("status") for item in entry.get("history", [])] != [ - "candidate" - ]: + elif statuses != ["candidate"]: problems.append( f"{name}: a new registration is added as a lone candidate, " "and moves on only in a later commit" @@ -565,15 +832,6 @@ def append_only_problems( return problems -def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: - return subprocess.run( - ["git", "-C", str(repository), *args], - capture_output=True, - text=True, - check=False, - ) - - def _commit(repository: Path, ref: str) -> str | None: found = _git( repository, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}" @@ -620,10 +878,9 @@ def base_commit(repository: Path) -> str: return base -def base_registry(repository: Path) -> dict[str, Any] | None: - """The registry at :func:`base_commit`, or ``None`` when the base has - no registry yet. Raises ``LookupError`` when no base resolves.""" - commit = base_commit(repository) +def registry_at(repository: Path, commit: str) -> dict[str, Any] | None: + """The registry at ``commit``, or ``None`` where it does not exist. + Raises ``LookupError`` when ``commit`` cannot be read.""" listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) if listed.returncode != 0: raise LookupError(f"cannot list {REGISTRY} at {commit}") @@ -633,3 +890,38 @@ def base_registry(repository: Path) -> dict[str, Any] | None: if shown.returncode != 0: raise LookupError(f"cannot read {REGISTRY} at {commit}") return json.loads(shown.stdout) + + +def base_registry(repository: Path) -> dict[str, Any] | None: + """The registry at :func:`base_commit`, or ``None`` when the base has + no registry yet. Raises ``LookupError`` when no base resolves.""" + return registry_at(repository, base_commit(repository)) + + +def history_problems(repository: Path, base: str) -> list[str]: + """Append-only problems between every pair of consecutive first-parent + commits from ``base`` to ``HEAD``, so a push of several commits cannot + hide a change that a later commit undoes.""" + listed = _git( + repository, "rev-list", "--first-parent", "--reverse", f"{base}..HEAD" + ) + if listed.returncode != 0: + raise LookupError(f"cannot list commits from {base} to HEAD") + problems: list[str] = [] + previous_commit = base + previous = registry_at(repository, base) + for commit in listed.stdout.split(): + current = registry_at(repository, commit) + if current is not None: + shape = validate(current, vocabulary=current.get("sources", {})) + if shape: + problems.extend(f"{commit[:12]}: {p}" for p in shape) + elif previous is not None: + problems.extend( + f"{previous_commit[:12]}..{commit[:12]}: {p}" + for p in append_only_problems(previous, current) + ) + elif previous is not None: + problems.append(f"{commit[:12]}: the registry was deleted") + previous_commit, previous = commit, current + return problems diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 63357dd8f..055a4943d 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,9 +39,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 10,020 | -| `artifact` | 4,253 | +| `unit` | 10,032 | +| `artifact` | 4,261 | | `integration_psid` | 1,450 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 308 | -| **Total** | **16,551** | +| **Total** | **16,571** | diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py index a8f58557d..247502c63 100644 --- a/tests/test_track_b_holdout_registry.py +++ b/tests/test_track_b_holdout_registry.py @@ -1,18 +1,17 @@ -"""The held-out data registry keeps its rules on every commit. - -``docs/design/track_b_holdout_registry.json`` is the single registry of -the data that Track B's gates and the open-population lane's gates hold -out (dynamics coordinator, 2026-10-21). -``populace_dynamics.track_b.holdout_registry`` states the rules; these -tests run them on the committed registry, against its base commit, and as -properties over invented registries. +"""The held-out data registry's rules, on invented registries. + +``populace_dynamics.track_b.holdout_registry`` states the rules of the +single held-out data registry (dynamics coordinator, 2026-10-21). These +tests exercise them on invented registries and invented git repositories, +with the expected families and overlaps worked out here, independently +of the module. The committed registry itself is checked in +``test_track_b_holdout_registry_committed.py``. """ from __future__ import annotations import copy import json -import os import subprocess from pathlib import Path @@ -22,8 +21,6 @@ from populace_dynamics.track_b import holdout_registry as registry -ROOT = Path(__file__).resolve().parents[1] - #: Specified here, independently of the module, so that dropping a field #: from the module's lists fails a test. EXPECTED_IMMUTABLE = ( @@ -43,70 +40,13 @@ "overlap_review", "history", ) - - -def committed() -> dict: - return registry.load(ROOT) - - -def by_id(current: dict) -> dict: - return {entry["id"]: entry for entry in current["entries"]} - - -# The committed registry ---------------------------------------------------- - - -def test_the_committed_registry_keeps_its_rules(): - assert registry.validate(committed(), ROOT) == [] - - -def test_the_committed_registry_only_appends_to_its_base(): - try: - base = registry.base_registry(ROOT) - except LookupError as error: - if os.environ.get("GITHUB_ACTIONS") == "true": - pytest.fail(f"CI must resolve the base registry: {error}") - pytest.skip(f"no base commit here: {error}") - if base is None: - return - assert registry.validate(base) == [] - assert registry.append_only_problems(base, committed()) == [] - - -def test_the_registry_stays_incomplete_until_the_inventory_lands(): - current = committed() - assert current["schema"] == registry.SCHEMA - if not (ROOT / current["completion_record"]).is_file(): - assert current["complete"] is False - assert any("incomplete" in rule for rule in current["rules"]) - - -@pytest.mark.parametrize( - "a, b", - [("H001", "H002"), ("H001", "H004"), ("H002", "H004"), ("H004", "H005")], -) -def test_the_seeded_shared_units_are_disclosed_both_ways(a, b): - entries = by_id(committed()) - assert registry.entries_overlap(entries[a], entries[b]) - assert b in entries[a]["disclosed_overlaps"] - assert a in entries[b]["disclosed_overlaps"] - - -def test_the_field_lists_are_as_specified(): - assert registry.IMMUTABLE == EXPECTED_IMMUTABLE - assert registry.EXTENDING == EXPECTED_EXTENDING - assert set(EXPECTED_IMMUTABLE) | set(EXPECTED_EXTENDING) | { - "status", - } == set(registry.ENTRY_KEYS) - - -# Invented registries ------------------------------------------------------- - -SOURCES = { +VOCABULARY = { "s_a": {"family": "f1", "title": "A", "catalog": "cat a", "key_year": "y"}, "s_b": {"family": "f1", "title": "B", "catalog": "cat b", "key_year": "y"}, "s_c": {"family": "f2", "title": "C", "catalog": "cat c", "key_year": "y"}, } +#: The family of each invented source, independently of the module. +FAMILY = {"s_a": "f1", "s_b": "f1", "s_c": "f2"} KEY_POOL = ( "s_a:2020", "s_a:2020-2022:x", @@ -123,14 +63,43 @@ def test_the_field_lists_are_as_specified(): ("registration", "used"): ["candidate", "frozen", "used"], ("registration", "withdrawn"): ["candidate", "withdrawn"], ("historical_import", "used"): ["used"], - ("historical_import", "frozen"): ["frozen"], - ("historical_import", "frozen_used"): ["frozen", "used"], } +REASON = ( + "The two units come from different years of separate sources, so no " + "observation is shared." +) -def make_entry( - index: int, keys: list[str], path=("registration", "candidate") -): +def run_path(name: str) -> str: + return f"{registry.RUNS_DIR}/{name}.json" + + +def key_years(key: str): + """The source and the inclusive year range of an invented key.""" + source, years = key.split(":")[:2] + if years == "undated": + return source, None + start, _, end = years.partition("-") + return source, (int(start), int(end or start)) + + +def share_unit(a: dict, b: dict) -> bool: + """Whether two entries share a unit, worked out independently.""" + for x in a["unit_keys"]: + for y in b["unit_keys"]: + (sx, yx), (sy, yy) = key_years(x), key_years(y) + if sx != sy: + continue + if yx is None or yy is None or (yx[0] <= yy[1] and yy[0] <= yx[1]): + return True + return False + + +def family_of(entry: dict) -> set[str]: + return {FAMILY[key_years(key)[0]] for key in entry["unit_keys"]} + + +def make_entry(index: int, keys, path=("registration", "candidate")): statuses = PATHS[path] gate = f"gate_{index}" return { @@ -145,13 +114,7 @@ def make_entry( "unit_keys": list(keys), "evidence": [f"evidence {index}"], "scored_evidence": ( - [ - { - "path": f"scores/s{index}.json", - "field": "gate", - "equals": gate, - } - ] + [{"path": run_path(f"s{index}"), "field": "gate", "equals": gate}] if statuses[-1] == "used" else [] ), @@ -164,13 +127,22 @@ def make_entry( } -def settle(entries: list[dict], sources=SOURCES) -> None: - """Disclose every shared unit both ways, and review every unshared +def review(entry_id: str) -> dict: + return { + "id": entry_id, + "outcome": "disjoint", + "reason": REASON, + "evidence": ["https://example.org/catalog"], + } + + +def settle(entries: list[dict]) -> None: + """Disclose every shared unit both ways and review every unshared same-family relation, as a careful registrant would.""" live = [entry for entry in entries if entry["status"] != "withdrawn"] for index, entry in enumerate(live): for other in live[index + 1 :]: - if registry.entries_overlap(entry, other): + if share_unit(entry, other): if other["id"] not in entry["disclosed_overlaps"]: entry["disclosed_overlaps"].append(other["id"]) if entry["id"] not in other["disclosed_overlaps"]: @@ -182,15 +154,12 @@ def settle(entries: list[dict], sources=SOURCES) -> None: for other in entries[:index]: if ( other["status"] != "withdrawn" - and registry.families(entry, sources) - & registry.families(other, sources) - and not registry.entries_overlap(entry, other) + and family_of(entry) & family_of(other) + and not share_unit(entry, other) and other["id"] not in entry["disclosed_overlaps"] and other["id"] not in reviewed ): - entry["overlap_review"].append( - {"id": other["id"], "reason": "different years"} - ) + entry["overlap_review"].append(review(other["id"])) def make_registry(entries: list[dict], complete=False) -> dict: @@ -200,13 +169,17 @@ def make_registry(entries: list[dict], complete=False) -> dict: "purpose": "invented", "rules": ["rule one", "the registry is incomplete"], "status_values": {status: status for status in registry.STATUSES}, - "sources": copy.deepcopy(SOURCES), + "sources": copy.deepcopy(VOCABULARY), "complete": complete, "completion_record": "docs/design/inventory.json", "entries": entries, } +def validate(current, repository=None): + return registry.validate(current, repository, vocabulary=VOCABULARY) + + entry_specs = st.lists( st.tuples( st.lists( @@ -214,12 +187,10 @@ def make_registry(entries: list[dict], complete=False) -> dict: ), st.sampled_from(sorted(PATHS)), ), - min_size=0, max_size=6, ) registration_specs = st.lists( st.lists(st.sampled_from(KEY_POOL), min_size=1, max_size=3, unique=True), - min_size=0, max_size=4, ) @@ -232,20 +203,28 @@ def build(specs, complete=False) -> dict: return make_registry(entries, complete) -def shared_pairs(entries): - live = [e for e in entries if e["status"] != "withdrawn"] - return [ - (a, b) - for i, a in enumerate(live) - for b in live[i + 1 :] - if registry.entries_overlap(a, b) - ] +def test_the_field_lists_are_as_specified(): + assert registry.IMMUTABLE == EXPECTED_IMMUTABLE + assert registry.EXTENDING == EXPECTED_EXTENDING + assert set(EXPECTED_IMMUTABLE) | set(EXPECTED_EXTENDING) | { + "status", + } == set(registry.ENTRY_KEYS) @settings(max_examples=200, deadline=None) @given(entry_specs) def test_settled_registries_are_valid(specs): - assert registry.validate(build(specs)) == [] + assert validate(build(specs)) == [] + + +@settings(max_examples=200, deadline=None) +@given(entry_specs) +def test_shared_units_match_the_independent_rule(specs): + current = build(specs) + entries = current["entries"] + for i, a in enumerate(entries): + for b in entries[i + 1 :]: + assert registry.entries_overlap(a, b) == share_unit(a, b) @settings(max_examples=200, deadline=None) @@ -256,7 +235,7 @@ def test_a_one_way_disclosure_is_refused(specs, data): if not disclosing: return data.draw(st.sampled_from(disclosing))["disclosed_overlaps"].pop() - problems = registry.validate(current) + problems = validate(current) assert any( "does not disclose it back" in p or "without disclosing" in p for p in problems @@ -269,44 +248,81 @@ def test_a_silent_shared_unit_is_refused(specs): current = build(specs) for entry in current["entries"]: entry["disclosed_overlaps"] = [] + entry["overlap_review"] = [] settle_reviews_only(current["entries"]) - problems = registry.validate(current) - shared = bool(shared_pairs(current["entries"])) + live = [e for e in current["entries"] if e["status"] != "withdrawn"] + shared = any( + share_unit(a, b) for i, a in enumerate(live) for b in live[i + 1 :] + ) + problems = validate(current) assert any("without disclosing" in p for p in problems) == shared def settle_reviews_only(entries): - for entry in entries: - entry["overlap_review"] = [] for index, entry in enumerate(entries): if entry["status"] == "withdrawn": continue for other in entries[:index]: if ( other["status"] != "withdrawn" - and registry.families(entry, SOURCES) - & registry.families(other, SOURCES) - and not registry.entries_overlap(entry, other) + and family_of(entry) & family_of(other) + and not share_unit(entry, other) ): - entry["overlap_review"].append( - {"id": other["id"], "reason": "different years"} - ) + entry["overlap_review"].append(review(other["id"])) @settings(max_examples=200, deadline=None) @given(entry_specs) def test_an_unreviewed_family_relation_is_refused(specs): current = build(specs) - entries = current["entries"] - needed = any(entry["overlap_review"] for entry in entries) - for entry in entries: + needed = any(entry["overlap_review"] for entry in current["entries"]) + for entry in current["entries"]: entry["overlap_review"] = [] - problems = registry.validate(current) + problems = validate(current) assert any("neither discloses nor reviews" in p for p in problems) == ( needed ) +def test_fixed_family_cases_follow_the_rule(): + # s_a and s_b share family f1 but no unit; s_c is another family. + same = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] + assert any( + "neither discloses nor reviews" in p + for p in validate(make_registry(copy.deepcopy(same))) + ) + reviewed = copy.deepcopy(same) + reviewed[1]["overlap_review"].append(review("H001")) + assert validate(make_registry(reviewed)) == [] + disclosed = copy.deepcopy(same) + disclosed[0]["disclosed_overlaps"].append("H002") + disclosed[1]["disclosed_overlaps"].append("H001") + assert validate(make_registry(disclosed)) == [] + other = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_c:2025"])] + assert validate(make_registry(other)) == [] + + +@pytest.mark.parametrize( + "change, message", + [ + ({"outcome": "unresolved"}, "is disclosed"), + ( + {"reason": "TODO: check later, once the inventory has landed."}, + "substantive", + ), + ({"reason": "short"}, "substantive"), + ({"evidence": []}, "cites nothing"), + ({"evidence": ["somewhere"]}, "not path:line or a URL"), + ({"evidence": ["docs/x.md:0"]}, "bad line range"), + ], +) +def test_a_review_needs_an_outcome_a_reason_and_citations(change, message): + entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] + entries[1]["overlap_review"].append(dict(review("H001"), **change)) + problems = validate(make_registry(entries)) + assert any(message in p for p in problems) + + @settings(max_examples=200, deadline=None) @given(entry_specs, registration_specs) def test_appending_lone_candidates_keeps_every_rule(base_specs, more): @@ -316,7 +332,7 @@ def test_appending_lone_candidates_keeps_every_rule(base_specs, more): for offset, keys in enumerate(more): current["entries"].append(make_entry(start + offset, keys)) settle(current["entries"]) - assert registry.validate(current) == [] + assert validate(current) == [] assert registry.append_only_problems(base, current) == [] @@ -334,7 +350,7 @@ def test_a_registration_cannot_arrive_already_frozen(base_specs, path): make_entry(len(current["entries"]), ["s_c:2099"], path) ) settle(current["entries"]) - assert registry.validate(current) == [] + assert validate(current) == [] assert any( "lone candidate" in p for p in registry.append_only_problems(base, current) @@ -439,14 +455,19 @@ def test_sources_rules_and_completeness_only_move_forward(): assert registry.append_only_problems(base, renamed) -def test_an_alias_source_is_refused(): +def test_the_registry_must_carry_the_controlled_vocabulary(): current = make_registry([]) - current["sources"]["s_d"] = dict(SOURCES["s_a"], catalog=" CAT A ") - assert any("alias" in p for p in registry.validate(current)) + current["sources"]["s_d"] = dict(VOCABULARY["s_c"], catalog="cat d") + assert any("controlled vocabulary" in p for p in validate(current)) + aliased = dict(VOCABULARY, s_d=dict(VOCABULARY["s_a"], catalog=" CAT A")) + problems = registry.validate( + dict(current, sources=aliased), vocabulary=aliased + ) + assert any("alias" in p for p in problems) @given( - st.sampled_from(sorted(SOURCES)), + st.sampled_from(sorted(VOCABULARY)), st.integers(1900, 2100), st.integers(0, 5), st.integers(1900, 2100), @@ -480,12 +501,12 @@ def test_keys_overlap_on_source_and_intersecting_years(s, y1, n1, y2, n2, x): def test_malformed_unit_keys_are_refused(key): entry = make_entry(0, ["s_a:2020"]) entry["unit_keys"] = [key] - assert registry.validate(make_registry([entry])) + assert validate(make_registry([entry])) def test_a_key_naming_no_source_is_refused(): entry = make_entry(0, ["s_z:2020"]) - problems = registry.validate(make_registry([entry])) + problems = validate(make_registry([entry])) assert any("names no source" in p for p in problems) @@ -494,6 +515,7 @@ def test_a_key_naming_no_source_is_refused(): [ ("registration", ["frozen"], "frozen", "starts as one of"), ("historical_import", ["candidate"], "candidate", "starts as one of"), + ("historical_import", ["frozen"], "frozen", "starts as one of"), ("registration", ["candidate", "used"], "used", "not a legal"), ("historical_import", ["used", "frozen"], "frozen", "not a legal"), ("registration", ["candidate", "frozen"], "candidate", "last status"), @@ -508,45 +530,51 @@ def test_status_follows_a_legal_history(provenance, statuses, last, message): entry["status"] = last if last == "used": entry["scored_evidence"] = [ - {"path": "s.json", "field": "gate", "equals": "gate_0"} + {"path": run_path("s"), "field": "gate", "equals": "gate_0"} ] - assert any(message in p for p in registry.validate(make_registry([entry]))) + assert any(message in p for p in validate(make_registry([entry]))) @pytest.mark.parametrize("date", ["2026-10-01", "2026-99-99", "yesterday"]) def test_history_dates_are_real_and_never_go_back(date): entry = make_entry(0, ["s_a:2020"], ("registration", "frozen")) entry["history"][1]["date"] = date - assert registry.validate(make_registry([entry])) + assert validate(make_registry([entry])) def test_repeated_ids_and_shared_gate_identifiers_are_refused(): current = build([(["s_a:2020"], ("registration", "candidate"))] * 2) current["entries"][1]["id"] = "H001" - assert any("ids repeat" in p for p in registry.validate(current)) + assert any("ids repeat" in p for p in validate(current)) current = build([(["s_a:2020"], ("registration", "candidate"))] * 2) current["entries"][1]["gate_identifiers"].append("gate_0") - assert any("already identifies" in p for p in registry.validate(current)) + assert any("already identifies" in p for p in validate(current)) -def test_scored_evidence_must_name_this_gate(): +@pytest.mark.parametrize( + "change, message", + [ + ({"equals": "gate_other"}, "does not identify this gate"), + ({"field": "title"}, "not an identity field"), + ({"path": "docs/design/registry.json"}, "is not runs/"), + ({"path": "../outside.json"}, "is not runs/"), + ], +) +def test_scored_evidence_must_be_a_run_artifact_naming_this_gate( + change, message +): entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) - entry["scored_evidence"][0]["equals"] = "gate_other" - problems = registry.validate(make_registry([entry])) - assert any("not this gate" in p for p in problems) + entry["scored_evidence"][0].update(change) + assert any(message in p for p in validate(make_registry([entry]))) entry["scored_evidence"] = [] - problems = registry.validate(make_registry([entry])) + problems = validate(make_registry([entry])) assert any("needs scored_evidence" in p for p in problems) def test_a_review_must_name_an_earlier_entry(): - current = build( - [(["s_a:2020"], ("registration", "candidate"))] - + [(["s_b:2025"], ("registration", "candidate"))] - ) - first = current["entries"][0] - first["overlap_review"].append({"id": "H002", "reason": "later"}) - problems = registry.validate(current) + entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_c:2025"])] + entries[0]["overlap_review"].append(review("H002")) + problems = validate(make_registry(entries)) assert any("not an earlier entry" in p for p in problems) @@ -578,7 +606,7 @@ def test_malformed_fields_are_reported_not_raised(path, value): for part in path[:-1]: target = target[part] target[path[-1]] = value - problems = registry.validate(current) + problems = validate(current) assert isinstance(problems, list) and problems @@ -591,19 +619,27 @@ def test_malformed_fields_are_reported_not_raised(path, value): @settings(max_examples=200, deadline=None) -@given(st.sampled_from(sorted(registry.ENTRY_KEYS) + ["entries"]), json_values) -def test_any_json_in_any_field_is_reported_not_raised(key, value): +@given( + st.sampled_from(sorted(registry.ENTRY_KEYS) + ["entries"]), + json_values, + st.booleans(), +) +def test_any_json_in_any_field_is_reported_not_raised(key, value, append): base = build([(["s_a:2020"], ("historical_import", "used"))]) current = copy.deepcopy(base) if key == "entries": current["entries"] = value + elif append: + added = make_entry(1, ["s_c:2099"]) + added[key] = value + current["entries"].append(added) else: current["entries"][0][key] = value - assert isinstance(registry.validate(current), list) + assert isinstance(validate(current), list) assert isinstance(registry.append_only_problems(base, current), list) -# Committed files and base commits, through git ---------------------------- +# Committed trees and base commits, through git ---------------------------- def git(repository: Path, *args: str) -> str: @@ -622,8 +658,13 @@ def init_repository(tmp_path: Path) -> Path: git(repository, "init", "-q", "-b", "master") git(repository, "config", "user.email", "t@example.com") git(repository, "config", "user.name", "t") - (repository / "README").write_text("r", encoding="utf-8") - git(repository, "add", "README") + write( + repository, + registry.GATES, + "gates:\n gate_0:\n id: zero_gate\n" + " registration_id: reg-zero\n", + ) + git(repository, "add", "-A") git(repository, "commit", "-q", "-m", "start") return repository @@ -635,77 +676,145 @@ def write(repository: Path, relative: str, document) -> None: path.write_text(text, encoding="utf-8") +def commit_all(repository: Path, message: str) -> str: + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", message) + return git(repository, "rev-parse", "HEAD").strip() + + @pytest.fixture def no_ci_env(monkeypatch): for name in ("GITHUB_BASE_REF", "GITHUB_EVENT_NAME", "GITHUB_EVENT_PATH"): monkeypatch.delenv(name, raising=False) -def test_scored_evidence_must_be_committed_and_identify_the_gate( - tmp_path, no_ci_env -): - repository = init_repository(tmp_path) +def scored_registry() -> dict: entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) - current = make_registry([entry]) - evidence = entry["scored_evidence"][0] + entry["gate_identifiers"] = ["gate_0", "zero_gate", "reg-zero"] + return make_registry([entry]) + + +def test_scored_evidence_is_read_from_the_committed_tree(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = scored_registry() + evidence = current["entries"][0]["scored_evidence"][0] write(repository, evidence["path"], {"gate": "gate_0"}) assert any( - "not committed" in p for p in registry.validate(current, repository) + "not a committed file" in p for p in validate(current, repository) ) git(repository, "add", "-A") + assert any( + "not a committed file" in p for p in validate(current, repository) + ) git(repository, "commit", "-q", "-m", "score") - assert registry.validate(current, repository) == [] - write(repository, "scores/other.json", {"gate": "gate_9"}) - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", "other") - evidence["path"] = "scores/other.json" + assert validate(current, repository) == [] + write(repository, evidence["path"], {"gate": "gate_9"}) + assert validate(current, repository) == [] + for field, equals in ( + ("registration_id", "reg-zero"), + ("gate", "zero_gate"), + ): + evidence.update(field=field, equals=equals) + assert any("does not have" in p for p in validate(current, repository)) + + +def test_identifiers_must_be_authoritative(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = scored_registry() + entry = current["entries"][0] + write(repository, entry["scored_evidence"][0]["path"], {"title": "x"}) + commit_all(repository, "score") + entry["gate_identifiers"].append("x") + entry["scored_evidence"][0].update(field="gate", equals="x") assert any( - "does not have gate" in p - for p in registry.validate(current, repository) + "does not identify this gate" in p + for p in validate(current, repository) ) - write(repository, "scores/text.json", "not json") - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", "text") - evidence["path"] = "scores/text.json" - assert any("not JSON" in p for p in registry.validate(current, repository)) + entry["gate"] = "gate_unknown" + entry["gate_identifiers"] = ["gate_unknown"] + entry["scored_evidence"][0]["equals"] = "gate_unknown" + assert any( + "neither a gates.yaml gate" in p for p in validate(current, repository) + ) + + +def test_a_symlinked_or_non_json_artifact_is_refused(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = scored_registry() + evidence = current["entries"][0]["scored_evidence"][0] outside = tmp_path / "outside.json" outside.write_text(json.dumps({"gate": "gate_0"}), encoding="utf-8") - (repository / "scores" / "link.json").symlink_to(outside) - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", "link") - evidence["path"] = "scores/link.json" - assert any("symlink" in p for p in registry.validate(current, repository)) - for path in ("../outside.json", "/etc/hosts", "~/x.json"): - evidence["path"] = path - assert any( - "outside the repository" in p - for p in registry.validate(current, repository) - ) + link = repository / evidence["path"] + link.parent.mkdir(parents=True) + link.symlink_to(outside) + commit_all(repository, "link") + assert any( + "not a committed file" in p for p in validate(current, repository) + ) + link.unlink() + write(repository, evidence["path"], "not json") + commit_all(repository, "text") + assert any("not JSON" in p for p in validate(current, repository)) -def test_complete_needs_its_completion_record(tmp_path, no_ci_env): +def test_imports_need_evidence_older_than_the_registry(tmp_path, no_ci_env): repository = init_repository(tmp_path) - current = make_registry([], complete=True) + current = scored_registry() + evidence = current["entries"][0]["scored_evidence"][0] + write(repository, evidence["path"], {"gate": "gate_0"}) + commit_all(repository, "score before the registry") + write(repository, registry.REGISTRY, current) + commit_all(repository, "registry") + assert validate(current, repository) == [] + late = make_entry(1, ["s_c:2099"], ("historical_import", "used")) + late["gate"] = "gate_0" + late["gate_identifiers"] = ["gate_0"] + late["scored_evidence"][0].update(path=run_path("late"), equals="gate_0") + write(repository, late["scored_evidence"][0]["path"], {"gate": "gate_0"}) + commit_all(repository, "score after the registry") + current["entries"].append(late) assert any( - "does not exist" in p for p in registry.validate(current, repository) + "after the registry existed" in p + for p in validate(current, repository) ) + + +def test_review_citations_must_name_committed_lines(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + write(repository, "docs/note.md", "one\ntwo\nthree\n") + commit_all(repository, "note") + entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] + entries[1]["overlap_review"].append(review("H001")) + current = make_registry(entries) + for citation, ok in ( + ("docs/note.md:2", True), + ("docs/note.md:2-3", True), + ("docs/note.md:4", False), + ("docs/missing.md:1", False), + ): + entries[1]["overlap_review"][0]["evidence"] = [citation] + assert (validate(current, repository) == []) == ok, citation + + +def test_complete_needs_its_committed_record(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = make_registry([], complete=True) + assert any("not committed" in p for p in validate(current, repository)) write(repository, current["completion_record"], {}) - assert registry.validate(current, repository) == [] + commit_all(repository, "inventory") + assert validate(current, repository) == [] def commit_registry(repository: Path, current: dict, message: str) -> str: write(repository, registry.REGISTRY, current) - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", message) - return git(repository, "rev-parse", "HEAD").strip() + return commit_all(repository, message) def test_a_pull_request_compares_with_its_merge_base(tmp_path, no_ci_env): repository = init_repository(tmp_path) git(repository, "checkout", "-q", "-b", "feature") write(repository, "notes.txt", "a commit of the branch's own") - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", "feature work") + commit_all(repository, "feature work") assert registry.base_registry(repository) is None git(repository, "checkout", "-q", "master") first = build([(["s_a:2020"], ("historical_import", "used"))]) @@ -753,6 +862,35 @@ def test_a_push_compares_with_its_before_commit( assert registry.base_registry(repository) == middle +@pytest.mark.parametrize("undo", ["append_then_remove", "edit_then_restore"]) +def test_a_push_cannot_hide_a_change_a_later_commit_undoes( + tmp_path, no_ci_env, undo +): + repository = init_repository(tmp_path) + first = build([(["s_a:2020"], ("historical_import", "used"))]) + base = commit_registry(repository, first, "registry") + changed = copy.deepcopy(first) + if undo == "append_then_remove": + changed["entries"].append(make_entry(1, ["s_c:2099"])) + commit_registry(repository, changed, "append") + else: + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "edit a key") + commit_registry(repository, first, "undo") + assert registry.append_only_problems(first, first) == [] + assert registry.history_problems(repository, base) + + +def test_a_clean_history_has_no_problems(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + first = build([(["s_a:2020"], ("historical_import", "used"))]) + base = commit_registry(repository, first, "registry") + second = copy.deepcopy(first) + second["entries"].append(make_entry(1, ["s_c:2099"])) + commit_registry(repository, second, "append a candidate") + assert registry.history_problems(repository, base) == [] + + def test_no_base_raises_rather_than_passing(tmp_path, monkeypatch): monkeypatch.delenv("GITHUB_EVENT_NAME", raising=False) monkeypatch.setenv("GITHUB_BASE_REF", "no-such-branch") diff --git a/tests/test_track_b_holdout_registry_committed.py b/tests/test_track_b_holdout_registry_committed.py new file mode 100644 index 000000000..6c7a6133e --- /dev/null +++ b/tests/test_track_b_holdout_registry_committed.py @@ -0,0 +1,155 @@ +"""The committed held-out data registry keeps its rules on every commit. + +``docs/design/track_b_holdout_registry.json`` is the single registry of +the data that Track B's gates and the open-population lane's gates hold +out (dynamics coordinator, 2026-10-21). These tests check the committed +registry against the committed tree, including its scored-run artifacts +under ``runs/``, against its base commit, and between every first-parent +commit since that base. The rules themselves are exercised on invented +registries in ``test_track_b_holdout_registry.py``. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from populace_dynamics.track_b import holdout_registry as registry + +ROOT = Path(__file__).resolve().parents[1] + +#: The seeded entries' unit keys, specified here independently. They are +#: immutable once merged: PSID by interview wave, EPUF by the vintage in +#: its title, publications by edition. +EXPECTED_SEED_KEYS = { + "H001": ["psid:2015-2023:held_out_window"], + "H002": ["psid:2013:fam2013er", "psid:2015:fam2015er"], + "H003": ["ssa_epuf:2006:test_partition"], + "H004": [ + "psid:1982:employment_status", + "psid:1983:employment_status", + "psid:1993-1997:employment_status", + "psid:1999-2023:employment_status", + "ssa_di_asr:2023", + ], + "H005": [ + "ssa_supplement:2023:6.b5.1", + "ssa_supplement:2023:6.b4", + "ssa_supplement:2023:6.b3", + "ssa_di_asr:2023:table_19", + "census_cps_tables:2023:hh-4", + "census_cps_tables:2023:ad-3", + "populace_us_frame:2024:certified", + "mermin_smith_anchors:undated", + ], +} +#: Every disclosed pair: four shared units and two unresolved relations +#: between EPUF and the 2023 DI report and Supplement tables. +EXPECTED_DISCLOSURES = { + ("H001", "H002"), + ("H001", "H004"), + ("H002", "H004"), + ("H003", "H004"), + ("H003", "H005"), + ("H004", "H005"), +} +EXPECTED_FAMILIES = { + "psid": "psid", + "ssa_epuf": "ssa_administrative", + "ssa_supplement": "ssa_administrative", + "ssa_di_asr": "ssa_administrative", + "census_cps_tables": "us_cross_section", + "populace_us_frame": "us_cross_section", + "mermin_smith_anchors": "literature", +} +EXPECTED_OUTSIDE_GATES_YAML = {"B2": ("B2", "6252388925")} + + +def committed() -> dict: + return registry.load(ROOT) + + +def by_id(current: dict) -> dict: + return {entry["id"]: entry for entry in current["entries"]} + + +def test_the_committed_registry_keeps_its_rules(): + assert registry.validate(committed(), ROOT) == [] + + +def resolved_base(): + try: + return registry.base_commit(ROOT) + except LookupError as error: + if os.environ.get("GITHUB_ACTIONS") == "true": + pytest.fail(f"CI must resolve the base commit: {error}") + pytest.skip(f"no base commit here: {error}") + + +def test_the_committed_registry_only_appends_to_its_base(): + base_commit = resolved_base() + base = registry.registry_at(ROOT, base_commit) + if base is not None: + assert registry.validate(base, vocabulary=base["sources"]) == [] + assert registry.append_only_problems(base, committed()) == [] + + +def test_every_first_parent_commit_since_the_base_appends_only(): + if os.environ.get("GITHUB_EVENT_NAME") != "push": + pytest.skip( + "a pull request is checked at its merge commit; a branch's own " + "draft commits are not the base branch's history" + ) + assert registry.history_problems(ROOT, resolved_base()) == [] + + +def test_the_registry_stays_incomplete_until_the_inventory_lands(): + current = committed() + assert current["schema"] == registry.SCHEMA + if not (ROOT / current["completion_record"]).is_file(): + assert current["complete"] is False + assert any("incomplete" in rule for rule in current["rules"]) + + +def test_the_seeded_unit_keys_are_as_specified(): + entries = by_id(committed()) + for name, keys in EXPECTED_SEED_KEYS.items(): + assert entries[name]["unit_keys"] == keys + + +def test_the_seeded_disclosures_are_as_specified(): + pairs = { + tuple(sorted((entry["id"], other))) + for entry in committed()["entries"] + for other in entry["disclosed_overlaps"] + } + assert pairs == EXPECTED_DISCLOSURES + entries = by_id(committed()) + for a, b in EXPECTED_DISCLOSURES - {("H003", "H004"), ("H003", "H005")}: + assert registry.entries_overlap(entries[a], entries[b]) + + +def test_the_vocabulary_and_registrations_are_as_specified(): + assert { + source: declared["family"] + for source, declared in registry.SOURCES.items() + } == EXPECTED_FAMILIES + assert dict(registry.REGISTRATIONS_OUTSIDE_GATES_YAML) == ( + EXPECTED_OUTSIDE_GATES_YAML + ) + assert committed()["sources"] == { + source: dict(declared) for source, declared in registry.SOURCES.items() + } + + +def test_every_used_entry_cites_committed_run_artifacts(): + for entry in committed()["entries"]: + if entry["status"] == "used": + assert entry["scored_evidence"], entry["id"] + for item in entry["scored_evidence"]: + relative = item["path"] + assert relative.startswith("runs/"), relative + assert relative.endswith(".json"), relative + assert (ROOT / relative).is_file(), relative diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 058ab50fc..dc36b664c 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 10020, - "artifact": 4253, + "unit": 10032, + "artifact": 4261, "integration_psid": 1450, "reproduction_legacy": 520, "oracle_policyengine": 308 From 296bf2bbd0e60374af985828b4e3a739450b1e9f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:19:33 -0400 Subject: [PATCH 4/9] Holdout registry: answer review round 3 (five medium, two low) 1. Preparation artifacts as scored evidence (medium). Scored runs are now a controlled list in code, SCORED_RUNS {path: (gate, sha256)}. Evidence must name a listed run that belongs to the entry's gate, and the committed blob must match the pinned SHA-256. Floor builders, candidate lists and dry runs are not listed, so they are rejected. 2. Import cutoff by pathname (medium). Historical imports are limited to HISTORICAL_IMPORT_GATES, a code-level list, and start as used. Content is pinned by hash, so renames, re-adds and merge-created registries cannot launder a future gate's evidence into an import. 3. Registration after scoring (medium). Committed.registered_at walks the registry's first-parent history; a registration must be on record at a commit where its scored run was not yet committed. 4. History checks (medium). history_problems validates each snapshot against its own commit's tree (evidence, citations, identifiers, completion record), admits a new registry against empty_like, and skips invalid snapshots as predecessors. 5. Rebase merging (medium). The committed module now checks every commit on the pull request branch (full=False), so a rebase cannot introduce an unchecked change and restoration. 6. Terminal-status mutant (low). The unit tests enumerate the permitted transitions independently (EXPECTED_TRANSITIONS) and check terminal statuses; used -> withdrawn is now killed. 7. Malformed inputs (low). Decoding, NUL and '..' paths, oversized line numbers, non-mapping registries and malformed predecessors return problems instead of raising. Seeded entries: H005 cites gates.yaml:4311-4327 for its Census labels; H003 cites the open-population design as PR #557 (draft). Mutations: 53 of 53 killed (restored source sha256 aae22b1d...). Tests: 98 passed, 2 skipped. Tier counts: unit 10,048 (+16), artifact 4,263 (+2). Co-Authored-By: Claude Opus 5.5 --- docs/design/track_b_holdout_registry.json | 13 +- .../track_b_holdout_registry_mutations.json | 143 ++++-- .../track_b/holdout_registry.py | 478 ++++++++++++++---- tests/test_track_b_holdout_registry.py | 371 ++++++++++---- ...test_track_b_holdout_registry_committed.py | 57 ++- tests/tier_counts.json | 4 +- 6 files changed, 827 insertions(+), 239 deletions(-) diff --git a/docs/design/track_b_holdout_registry.json b/docs/design/track_b_holdout_registry.json index a56381dcc..bac6d505b 100644 --- a/docs/design/track_b_holdout_registry.json +++ b/docs/design/track_b_holdout_registry.json @@ -3,15 +3,15 @@ "title": "Held-out data registry for Track B and the open-population lane", "purpose": "The single registry of the data that every Track B gate and every open-population gate holds out, set up by the dynamics coordinator on 2026-10-21. It is part of Track B's B0.2 exposure record (design revision 3, section 5.2, elements 4 and 5). Cite it by path, commit and SHA-256. src/populace_dynamics/track_b/holdout_registry.py states and enforces its rules, and tests/test_track_b_holdout_registry.py runs them on every commit.", "rules": [ - "A gate registers what it holds out here, as a lone candidate entry in a commit of its own, before its record freezes. It moves to frozen, used or withdrawn only in later commits.", + "A gate registers what it holds out here, as a lone candidate entry in a commit of its own, before its record freezes and before its scored run is committed. It moves to frozen, used or withdrawn only in later commits.", "J's candidate holdouts are registered first, because J is on Track B's critical path. OP0.2 registers its picks for OP12's gate_ssa_totals here before it freezes them, and picks from the years J has not taken.", "Every unit key names a source from the controlled vocabulary in src/populace_dynamics/track_b/holdout_registry.py (SOURCES), then its years (a year, an inclusive range, or 'undated'), then optional scope segments. The registry's sources must equal that vocabulary. Each source declares a family, a title, a catalog reference no other source shares, and what its key year means. Adding a source is a reviewed code change, and a new source must not be another name for a declared one.", "Two unit keys share a unit when their source is the same and their years intersect; scope segments describe the unit and never narrow the overlap. Live entries that share a unit, or whose relation is unresolved, list each other in disclosed_overlaps, and both gates' registrations name the overlap. No unit is held out by two gates silently.", "The overlap rule cannot see two sources that share underlying data, or two editions of a publication that report the same data years. So an entry reviews every earlier live entry of the same source family that it neither shares a unit with nor discloses. A review records the outcome 'disjoint', a substantive reason and citations (path:line or https URLs); an unresolved relation is disclosed instead.", "Reference years are recorded separately from interview or publication availability (design section 5.2, element 10): a unit key's year is what its source declares (for PSID, the interview wave; for EPUF, the file's vintage year), and each entry's units state the reference or event years.", - "An entry whose gate has scored has status 'used'. Its scored_evidence cites committed scored-run artifacts under runs/, each with an identity field (gate, gate.name or registration_id) whose value identifies the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", - "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused. These rules hold against the base commit and between every pair of consecutive first-parent commits since it.", - "A historical import records a gate that scored before this registry existed: it starts as used, and its scored evidence must have been committed before the registry was. Imports are allowed only while the registry is incomplete.", + "An entry whose gate has scored has status 'used'. Its scored_evidence cites scored-run artifacts listed in the module's SCORED_RUNS, which pins each artifact's gate and the SHA-256 of its bytes; an artifact joins that list, as a reviewed code change, when its gate scores. Each citation also names an identity field (gate, gate.name or registration_id) whose value identifies the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. A registration's scored run must not have been committed when the entry was registered. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", + "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused. These rules hold against the base commit, between every pair of consecutive first-parent commits on the base branch (each snapshot also validated against its own commit's tree), and between consecutive commits of a pull request's branch. A newly created registry is admitted against an empty one.", + "A historical import records a gate that scored before this registry existed. It starts as used, and its gate must be in the module's HISTORICAL_IMPORT_GATES, which B0.2's mechanical inventory may extend by a reviewed code change. Imports are allowed only while the registry is incomplete.", "The registry is incomplete until completion_record, B0.2's mechanical exposure inventory, exists. It starts with the prior uses verified below; the inventory adds the rest before any B0.2 gate record freezes." ], "status_values": { @@ -232,7 +232,7 @@ { "date": "2026-10-21", "status": "used", - "note": "seeded from gates.yaml at 1e739d32; its relation to the DI ASR 2023 (H004, H005) is unresolved and disclosed; open_population.md rev 2 plans gate_young_earn on a new salt of the same file, which shares this unit and must disclose it" + "note": "seeded from gates.yaml at 1e739d32; its relation to the DI ASR 2023 (H004, H005) is unresolved and disclosed; the open-population design (PR #557, draft) plans gate_young_earn on a new salt of the same file, which shares this unit and must disclose it" } ] }, @@ -331,7 +331,8 @@ ], "evidence": [ "gates.yaml:3735-3736 (holdout_basis)", - "gates.yaml:4582-4584 (sources)" + "gates.yaml:4582-4584 (sources)", + "gates.yaml:4311-4327 (the Census HH-4 and AD-3 2023 sources, recorded as report-only)" ], "scored_evidence": [ { diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json index 199fa932c..693c8f70e 100644 --- a/docs/design/track_b_holdout_registry_mutations.json +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -4,7 +4,7 @@ "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", "mutation_method": "One deliberate production-line change per check; both registry test modules run in full and are expected to fail; original bytes restored before the next check. No mutated implementation is committed.", "restored_source_sha256": { - "src/populace_dynamics/track_b/holdout_registry.py": "b1064e0fe899556110f8eccb9c4cf914d2dd2fdee8c455c0b49d88cd73b2026c" + "src/populace_dynamics/track_b/holdout_registry.py": "aae22b1de559901fd563897bf6f7ce7296f1cc00caa46e32163cbe89ab6997f6" }, "checks": [ { @@ -36,6 +36,7 @@ "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen", + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", "tests/test_track_b_holdout_registry.py::test_an_unreviewed_family_relation_is_refused", "tests/test_track_b_holdout_registry.py::test_appending_lone_candidates_keeps_every_rule", "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", @@ -63,11 +64,22 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", "tests/test_track_b_holdout_registry_committed.py::test_the_seeded_disclosures_are_as_specified" ] }, + { + "mutation": "paths may climb out", + "old": "and \"..\" not in path.split(\"/\")", + "new": "and True", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_committed_record" + ] + }, { "mutation": "symlinks read as committed files", "old": "and fields[0] in (\"100644\", \"100755\")", @@ -78,6 +90,16 @@ "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused" ] }, + { + "mutation": "any gate may be imported", + "old": "and entry[\"gate\"] not in import_gates", + "new": "and False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_only_listed_gates_may_be_imported" + ] + }, { "mutation": "unknown gates accepted", "old": "if not authoritative:", @@ -99,14 +121,35 @@ ] }, { - "mutation": "evidence outside runs/ accepted", - "old": "if not RUN_PATH_RX.fullmatch(path):", + "mutation": "unregistered registrations may cite scores", + "old": "if registered is None:", "new": "if False:", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-is not runs/]", - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-is not runs/]" + "tests/test_track_b_holdout_registry.py::test_a_used_registration_must_already_be_registered" + ] + }, + { + "mutation": "any runs file counts as scored", + "old": "if run is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses3-used-not a legal]" + ] + }, + { + "mutation": "another gate's run accepted", + "old": "if run[0] != entry[\"gate\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change4-'gate_1''s scored run]" ] }, { @@ -131,14 +174,13 @@ ] }, { - "mutation": "evidence need not be committed", - "old": "if text is None:\n problems.append(\n f\"{name}: scored evidence {path!r} is not a committed file\"", - "new": "if False:\n problems.append(\n f\"{name}: scored evidence {path!r} is not a committed file\"", + "mutation": "pinned bytes unchecked", + "old": "if hashlib.sha256(content).hexdigest() != run[1]:", + "new": "if False:", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused", - "tests/test_track_b_holdout_registry.py::test_scored_evidence_is_read_from_the_committed_tree" + "tests/test_track_b_holdout_registry.py::test_a_scored_run_must_keep_its_pinned_bytes" ] }, { @@ -148,17 +190,18 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_symlinked_or_non_json_artifact_is_refused", "tests/test_track_b_holdout_registry.py::test_scored_evidence_is_read_from_the_committed_tree" ] }, { - "mutation": "late imports accepted", - "old": "if added is None or not committed.is_ancestor(added, cutoff):", - "new": "if False:", + "mutation": "registration may follow its score", + "old": "if earlier is not None and (", + "new": "if False and (", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_imports_need_evidence_older_than_the_registry" + "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]" ] }, { @@ -198,7 +241,8 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change4-not path:line or a URL]" + "tests/test_track_b_holdout_registry.py::test_a_review_needs_an_outcome_a_reason_and_citations[change4-not path:line or a URL]", + "tests/test_track_b_holdout_registry.py::test_review_citations_must_name_committed_text_lines" ] }, { @@ -208,7 +252,17 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_review_citations_must_name_committed_lines" + "tests/test_track_b_holdout_registry.py::test_review_citations_must_name_committed_text_lines" + ] + }, + { + "mutation": "completion record path unchecked", + "old": "if not safe_path(record):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_complete_needs_its_committed_record" ] }, { @@ -263,7 +317,11 @@ "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[historical_import-statuses4-frozen-not a legal]", - "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses3-used-not a legal]" + "tests/test_track_b_holdout_registry.py::test_status_follows_a_legal_history[registration-statuses3-used-not a legal]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses0]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses1]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses2]", + "tests/test_track_b_holdout_registry.py::test_terminal_and_repeated_statuses_are_refused[statuses3]" ] }, { @@ -285,8 +343,9 @@ "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change0-does not identify this gate]", "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change1-not an identity field]", - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-is not runs/]", - "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-is not runs/]" + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change2-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change3-not a listed scored run]", + "tests/test_track_b_holdout_registry.py::test_scored_evidence_must_be_a_run_artifact_naming_this_gate[change4-'gate_1''s scored run]" ] }, { @@ -299,20 +358,9 @@ "tests/test_track_b_holdout_registry.py::test_the_registry_must_carry_the_controlled_vocabulary" ] }, - { - "mutation": "registry may carry its own vocabulary", - "old": "if registry[\"sources\"] != dict(vocabulary):", - "new": "if False:", - "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", - "exit_code": 1, - "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_malformed_fields_are_reported_not_raised[path5-]", - "tests/test_track_b_holdout_registry.py::test_the_registry_must_carry_the_controlled_vocabulary" - ] - }, { "mutation": "complete needs no record", - "old": "and committed.text(record) is None", + "old": "and committed.blob(record) is None", "new": "and False", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, @@ -421,8 +469,10 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_push_cannot_hide_a_change_a_later_commit_undoes[append_then_remove]", "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_any_json_in_any_field_is_reported_not_raised", "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused", "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" ] @@ -434,7 +484,8 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_push_cannot_hide_a_change_a_later_commit_undoes[edit_then_restore]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-edit_then_restore]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-edit_then_restore]", "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused" ] }, @@ -465,6 +516,7 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_new_registry_is_admitted_against_an_empty_one", "tests/test_track_b_holdout_registry.py::test_a_registration_cannot_arrive_already_frozen" ] }, @@ -497,10 +549,31 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_clean_history_has_no_problems", "tests/test_track_b_holdout_registry.py::test_a_pull_request_compares_with_its_merge_base", "tests/test_track_b_holdout_registry_committed.py::test_the_committed_registry_only_appends_to_its_base" ] }, + { + "mutation": "snapshots not validated in history", + "old": "if full:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_each_snapshot_is_validated_against_its_own_tree" + ] + }, + { + "mutation": "a new registry needs no admission", + "old": "before = previous if previous is not None else empty_like(current)", + "new": "before = previous if previous is not None else current", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_new_registry_is_admitted_against_an_empty_one" + ] + }, { "mutation": "history checks only the endpoint", "old": "previous_commit, previous = commit, current", @@ -508,7 +581,9 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_push_cannot_hide_a_change_a_later_commit_undoes[append_then_remove]" + "tests/test_track_b_holdout_registry.py::test_a_clean_history_has_no_problems", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]" ] } ] diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py index 0645033e1..4a715dc1c 100644 --- a/src/populace_dynamics/track_b/holdout_registry.py +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -25,15 +25,20 @@ - unique entry ids, which are never removed, reordered or reused; - histories with legal transitions only. A registration starts as a lone ``candidate`` in a commit of its own, so it reaches the base branch - before its gate freezes. A historical import is a gate that scored - before the registry existed: it starts as ``used``, and its scored - evidence must have been committed before the registry was; -- a ``used`` entry cites scored-run artifacts under ``runs/``, read from - the committed tree, whose identity field equals one of the gate's - identifiers in ``gates.yaml`` or :data:`REGISTRATIONS_OUTSIDE_GATES_YAML`; -- entries only grow: against the base registry, and between every pair - of consecutive first-parent commits since it, an entry's identity and - units never change, and its lists only extend; + before its gate freezes, and its scored run must not yet be committed + when it is registered. A historical import is a gate that scored before + the registry existed: it starts as ``used``, and its gate must be in + :data:`HISTORICAL_IMPORT_GATES`; +- a ``used`` entry cites scored-run artifacts listed in + :data:`SCORED_RUNS`, which pins each one's gate and the SHA-256 of its + bytes, read from the committed tree; each citation's identity field + names the gate in ``gates.yaml`` or in + :data:`REGISTRATIONS_OUTSIDE_GATES_YAML`; +- entries only grow: against the base registry, between every pair of + consecutive first-parent commits since it (each snapshot validated + against its own commit's tree), and between a pull request's own + commits, an entry's identity and units never change and its lists only + extend. A newly created registry is admitted against an empty one; - the registry stays incomplete until its completion record, B0.2's mechanical exposure inventory, exists. @@ -51,6 +56,7 @@ from __future__ import annotations import datetime as dt +import hashlib import json import os import re @@ -155,6 +161,61 @@ REGISTRATIONS_OUTSIDE_GATES_YAML: Mapping[str, tuple[str, ...]] = ( MappingProxyType({"B2": ("B2", "6252388925")}) ) +#: Each scored-run artifact a ``used`` entry may cite: its gate and the +#: SHA-256 of its committed bytes. A gate's scored run is added here when +#: it scores, as a reviewed code change, so a floor, candidate file or +#: dry run cannot stand in for a score, and a reused path cannot carry +#: another gate's run. +SCORED_RUNS: Mapping[str, tuple[str, str]] = MappingProxyType( + { + "runs/gate_m6_candidate1_v1.json": ( + "gate_m6", + "546a9739f8d1c7d21a91a07eb902c8af9bda92cdaa8f7917f312894f6a861b24", + ), + "runs/gate_m6_candidate2_v1.json": ( + "gate_m6", + "f217b0dc3b17b0a2c2926b1e9299ba2144be4b8a69ac880566d19fd4cd6c2917", + ), + "runs/gate_m6_candidate3_v1.json": ( + "gate_m6", + "caf254925f44b27c1bd1131336055e27fbc311daec00b0050b2f9293a74e82cf", + ), + "runs/track_b_b2_score.json": ( + "B2", + "df82a102a2b4c40c34f0e36b4333dbe7b90d13bfb3658b60b3f6d3f3e2dd8489", + ), + "runs/epuf_fill_gate_test_v1.json": ( + "gate_epuf_fill", + "9fa726be90134546c54148d681209ccf4a33939a7d6e2cfe5c2e384832dace03", + ), + "runs/gate_m4_hazard_v1.json": ( + "gate_m4", + "6493fda8207e743b3b6cbe1c80eb5ab5fb9646aa6f10fad4e77aa8d84c6b889c", + ), + "runs/gate_w1_candidate1_v1.json": ( + "gate_w1", + "01130c512ddba545e11dcf83364387a6c8df307041ac199ce3703cc4a00d6725", + ), + "runs/gate_w1_candidate2_v1.json": ( + "gate_w1", + "3baf097284b793fa4968d9b0b935ec3d16cf2770b16fb1fc1e3fb591ad490f40", + ), + "runs/gate_w1_candidate3_v1.json": ( + "gate_w1", + "b532f2b75d96e876af2e65400439a2f6c832ea45eecf83c1b4431b026ab7e688", + ), + "runs/gate_w1_candidate4_v1.json": ( + "gate_w1", + "ce4f9c9ab0b78bbb4dfe7dcb2a6b34892a80ea74760e0a00c4d60aa35e2d6e65", + ), + } +) +#: The gates that scored before the registry existed and may enter it as +#: historical imports. B0.2's mechanical inventory adds any it finds, as a +#: reviewed code change; no other gate can import itself. +HISTORICAL_IMPORT_GATES = frozenset( + {"gate_m6", "B2", "gate_epuf_fill", "gate_m4", "gate_w1"} +) STATUSES = ("candidate", "frozen", "used", "withdrawn") TRANSITIONS = { "candidate": frozenset({"frozen", "withdrawn"}), @@ -226,9 +287,11 @@ #: Scored-run artifacts live here. RUNS_DIR = "runs" RUN_PATH_RX = re.compile(rf"{RUNS_DIR}/[A-Za-z0-9_.\-]+\.json") +#: A repository-relative path: no leading slash, no NUL, no ``..``. +PATH_RX = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_./\-]*") CITATION_RX = re.compile( - r"(?P[A-Za-z0-9_][A-Za-z0-9_./\-]*):(?P[0-9]+)" - r"(?:-(?P[0-9]+))?" + r"(?P[A-Za-z0-9_][A-Za-z0-9_./\-]*):(?P[0-9]{1,7})" + r"(?:-(?P[0-9]{1,7}))?" ) URL_RX = re.compile(r"https://\S+") SHA_RX = re.compile(r"[0-9a-f]{40}") @@ -313,42 +376,83 @@ def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: ) +def _git_bytes(repository: Path, *args: str) -> bytes | None: + found = subprocess.run( + ["git", "-C", str(repository), *args], + capture_output=True, + check=False, + ) + return found.stdout if found.returncode == 0 else None + + +def safe_path(path: Any) -> bool: + """Whether ``path`` is a plain repository-relative path.""" + return ( + isinstance(path, str) + and PATH_RX.fullmatch(path) is not None + and ".." not in path.split("/") + ) + + +def _parse_registry(text: str | None) -> Any: + """A registry document, or ``None`` when ``text`` is not JSON.""" + if text is None: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + class Committed: """Read-only access to one commit of ``repository``, cached.""" def __init__(self, repository: Path, commit: str = "HEAD") -> None: self.repository = repository self.commit = commit - self._files: dict[str, str | None] = {} + self._files: dict[str, bytes | None] = {} self._gates: Mapping[str, Any] | None = None - self._first_added: dict[str, str | None] = {} + self._registered: dict[str, str | None] | None = None + + def blob(self, path: str, commit: str | None = None) -> bytes | None: + """A regular committed file's bytes at ``commit`` (by default this + one's), or ``None`` for a symlink, a directory, a missing path or + an unsafe one.""" + commit = commit or self.commit + key = f"{commit}:{path}" + if key not in self._files: + content = None + if safe_path(path): + listed = _git(self.repository, "ls-tree", commit, "--", path) + fields = listed.stdout.split() + if ( + listed.returncode == 0 + and len(fields) >= 4 + and fields[0] in ("100644", "100755") + and fields[1] == "blob" + ): + content = _git_bytes(self.repository, "show", key) + self._files[key] = content + return self._files[key] def text(self, path: str) -> str | None: - """A regular committed file's text, or ``None``.""" - if path not in self._files: - listed = _git(self.repository, "ls-tree", self.commit, "--", path) - fields = listed.stdout.split() - regular = ( - listed.returncode == 0 - and len(fields) >= 4 - and fields[0] in ("100644", "100755") - and fields[1] == "blob" - ) - shown = ( - _git(self.repository, "show", f"{self.commit}:{path}") - if regular - else None - ) - self._files[path] = ( - shown.stdout if shown and shown.returncode == 0 else None - ) - return self._files[path] + """A regular committed UTF-8 file's text, or ``None``.""" + content = self.blob(path) + if content is None: + return None + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return None def identifiers(self, gate: str) -> set[str]: """Every identifier that may name ``gate`` in scored evidence.""" if self._gates is None: text = self.text(GATES) - document = yaml.safe_load(text) if text else {} + try: + document = yaml.safe_load(text) if text else {} + except yaml.YAMLError: + document = {} gates = ( document.get("gates") if isinstance(document, dict) else None ) @@ -362,56 +466,101 @@ def identifiers(self, gate: str) -> set[str]: names.add(declared[key]) return names - def first_added(self, path: str) -> str | None: - """The oldest commit before ``self.commit`` that added ``path``.""" - if path not in self._first_added: + def registered_at(self, entry_id: str) -> str | None: + """The oldest commit on this commit's first-parent chain whose + registry holds ``entry_id``, or ``None``.""" + if self._registered is None: + self._registered = {} log = _git( self.repository, "log", - "--diff-filter=A", + "--first-parent", + "--reverse", "--format=%H", self.commit, "--", - path, + REGISTRY, ) - commits = log.stdout.split() if log.returncode == 0 else [] - self._first_added[path] = commits[-1] if commits else None - return self._first_added[path] - - def is_ancestor(self, older: str, newer: str) -> bool: - return ( - _git( - self.repository, "merge-base", "--is-ancestor", older, newer - ).returncode - == 0 - ) + for commit in log.stdout.split() if log.returncode == 0 else []: + document = _parse_registry( + _decode( + _git_bytes( + self.repository, "show", f"{commit}:{REGISTRY}" + ) + ) + ) + entries = ( + document.get("entries") + if isinstance(document, Mapping) + else None + ) + for entry in entries if isinstance(entries, list) else []: + name = ( + entry.get("id") if isinstance(entry, Mapping) else None + ) + if isinstance(name, str): + self._registered.setdefault(name, commit) + return self._registered.get(entry_id) + + +def _decode(content: bytes | None) -> str | None: + if content is None: + return None + try: + return content.decode("utf-8") + except UnicodeDecodeError: + return None def _evidence_problems( - entry: Mapping[str, Any], committed: Committed | None + entry: Mapping[str, Any], + committed: Committed | None, + scored_runs: Mapping[str, tuple[str, str]], + import_gates: frozenset[str], ) -> list[str]: - """Problems with an entry's scored-evidence citations.""" - if not entry["scored_evidence"]: - return [] + """Problems with an entry's scored-evidence citations and, for a + historical import, with its gate.""" name = entry["id"] problems: list[str] = [] + if ( + entry["provenance"] == "historical_import" + and entry["gate"] not in import_gates + ): + problems.append( + f"{name}: {entry['gate']!r} is not a gate that scored before the " + "registry existed" + ) + if not entry["scored_evidence"]: + return problems allowed = set(entry["gate_identifiers"]) if committed is not None: authoritative = committed.identifiers(entry["gate"]) if not authoritative: return [ + *problems, f"{name}: {entry['gate']!r} is neither a gates.yaml gate nor " - "a registration this module lists" + "a registration this module lists", ] allowed &= authoritative - cutoff = None - if committed is not None and entry["provenance"] == "historical_import": - cutoff = committed.first_added(REGISTRY) + registered = None + if committed is not None and entry["provenance"] == "registration": + registered = committed.registered_at(name) + if registered is None: + problems.append( + f"{name}: a registration cites scored evidence before it is " + "registered on the base branch" + ) for item in entry["scored_evidence"]: path, field, equals = item["path"], item["field"], item["equals"] - if not RUN_PATH_RX.fullmatch(path): + run = scored_runs.get(path) + if run is None: problems.append( - f"{name}: scored evidence {path!r} is not runs/*.json" + f"{name}: scored evidence {path!r} is not a listed scored run" + ) + continue + if run[0] != entry["gate"]: + problems.append( + f"{name}: scored evidence {path!r} is {run[0]!r}'s scored run" ) continue if field not in IDENTITY_FIELDS: @@ -425,29 +574,33 @@ def _evidence_problems( continue if committed is None: continue - text = committed.text(path) - if text is None: + content = committed.blob(path) + if content is None: problems.append( f"{name}: scored evidence {path!r} is not a committed file" ) continue - try: - document = json.loads(text) - except json.JSONDecodeError: - problems.append(f"{name}: scored evidence {path!r} is not JSON") + if hashlib.sha256(content).hexdigest() != run[1]: + problems.append( + f"{name}: scored evidence {path!r} differs from the listed " + "scored run's bytes" + ) continue + document = _parse_registry(_decode(content)) if lookup(document, field) != equals: problems.append( f"{name}: scored evidence {path!r} does not have " f"{field} == {equals!r}" ) continue - if cutoff is not None: - added = committed.first_added(path) - if added is None or not committed.is_ancestor(added, cutoff): + if registered is not None: + earlier = committed.blob(path, registered) + if earlier is not None and ( + hashlib.sha256(earlier).hexdigest() == run[1] + ): problems.append( - f"{name}: a historical import's evidence {path!r} was " - "committed after the registry existed" + f"{name}: its scored run {path!r} was already committed " + "when the entry was registered" ) return problems @@ -546,6 +699,8 @@ def _entry_problems( entry: Mapping[str, Any], sources: Mapping[str, Any], committed: Committed | None, + scored_runs: Mapping[str, tuple[str, str]], + import_gates: frozenset[str], ) -> list[str]: """Problems inside one well-shaped entry.""" name = entry["id"] @@ -605,7 +760,9 @@ def _entry_problems( ) if entry["status"] == "used" and not entry["scored_evidence"]: problems.append(f"{name}: a used entry needs scored_evidence") - problems.extend(_evidence_problems(entry, committed)) + problems.extend( + _evidence_problems(entry, committed, scored_runs, import_gates) + ) problems.extend(_review_problems(entry, committed)) return problems @@ -639,11 +796,17 @@ def validate( registry: Any, repository: Path | None = None, vocabulary: Mapping[str, Mapping[str, str]] = SOURCES, + *, + commit: str = "HEAD", + scored_runs: Mapping[str, tuple[str, str]] = SCORED_RUNS, + import_gates: frozenset[str] = HISTORICAL_IMPORT_GATES, ) -> list[str]: """Every rule one registry must keep on its own. ``vocabulary`` is the - controlled source list the registry's copy must equal. With - ``repository``, scored evidence, citations and the completion record - are checked against its ``HEAD`` commit.""" + controlled source list the registry's copy must equal; + ``scored_runs`` and ``import_gates`` are the controlled lists of + scored runs and historical-import gates. With ``repository``, scored + evidence, citations and the completion record are checked against its + ``commit``.""" if not isinstance(registry, Mapping): return ["registry: not a mapping"] missing = [key for key in TOP_LEVEL if key not in registry] @@ -673,7 +836,9 @@ def validate( if not isinstance(registry["complete"], bool): problems.append("registry: complete must be true or false") problems.extend(_vocabulary_problems(vocabulary)) - if registry["sources"] != dict(vocabulary): + if not isinstance(vocabulary, Mapping) or registry["sources"] != dict( + vocabulary + ): problems.append( "registry: sources must equal the controlled vocabulary" ) @@ -686,18 +851,26 @@ def validate( problems.extend(_shape_problems(entry)) if problems: return problems - committed = Committed(repository) if repository is not None else None + committed = ( + Committed(repository, commit) if repository is not None else None + ) record = registry["completion_record"] + if not safe_path(record): + return [f"registry: completion_record {record!r} is not a path"] if ( committed is not None and registry["complete"] - and committed.text(record) is None + and committed.blob(record) is None ): problems.append( f"registry: marked complete, but {record} is not committed" ) for entry in entries: - problems.extend(_entry_problems(entry, vocabulary, committed)) + problems.extend( + _entry_problems( + entry, vocabulary, committed, scored_runs, import_gates + ) + ) if problems: return problems ids = [entry["id"] for entry in entries] @@ -770,11 +943,27 @@ def _extends(base: Any, current: Any) -> bool: ) +def empty_like(registry: Mapping[str, Any]) -> dict[str, Any]: + """The registry as it stood before its first entry: the base a newly + created registry is admitted against.""" + return { + "schema": registry.get("schema"), + "status_values": registry.get("status_values"), + "completion_record": registry.get("completion_record"), + "rules": [], + "sources": {}, + "complete": False, + "entries": [], + } + + def append_only_problems( base: Mapping[str, Any], current: Mapping[str, Any] ) -> list[str]: """How ``current`` breaks the append-only rule against ``base``, an earlier registry that kept the rules.""" + if not isinstance(base, Mapping) or not isinstance(current, Mapping): + return ["registry: an append-only comparison needs two mappings"] problems: list[str] = [] for key in ("schema", "status_values", "completion_record"): if current.get(key) != base.get(key): @@ -784,17 +973,21 @@ def append_only_problems( sources = current.get("sources") if not isinstance(sources, Mapping): sources = {} - for source, declared in base.get("sources", {}).items(): + base_sources = base.get("sources") + if not isinstance(base_sources, Mapping): + base_sources = {} + for source, declared in base_sources.items(): if sources.get(source) != declared: problems.append(f"source {source!r}: changed or removed") if base.get("complete") is True and current.get("complete") is not True: problems.append("registry: a complete registry became incomplete") - old = base.get("entries", []) + old = base.get("entries") new = current.get("entries") - if not isinstance(new, list) or not all( - isinstance(entry, Mapping) for entry in new - ): - return [*problems, "registry: entries are malformed"] + for entries in (old, new): + if not isinstance(entries, list) or not all( + isinstance(entry, Mapping) for entry in entries + ): + return [*problems, "registry: entries are malformed"] old_ids = [entry.get("id") for entry in old] new_ids = [entry.get("id") for entry in new] if new_ids[: len(old_ids)] != old_ids: @@ -878,50 +1071,115 @@ def base_commit(repository: Path) -> str: return base -def registry_at(repository: Path, commit: str) -> dict[str, Any] | None: - """The registry at ``commit``, or ``None`` where it does not exist. - Raises ``LookupError`` when ``commit`` cannot be read.""" +def registry_at(repository: Path, commit: str) -> Any: + """The registry document at ``commit``, or ``None`` where it does not + exist. Raises ``LookupError`` when ``commit`` cannot be read and + ``ValueError`` when the registry there is not JSON.""" listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) if listed.returncode != 0: raise LookupError(f"cannot list {REGISTRY} at {commit}") if not listed.stdout.strip(): return None - shown = _git(repository, "show", f"{commit}:{REGISTRY}") - if shown.returncode != 0: - raise LookupError(f"cannot read {REGISTRY} at {commit}") - return json.loads(shown.stdout) + document = _parse_registry( + _decode(_git_bytes(repository, "show", f"{commit}:{REGISTRY}")) + ) + if document is None: + raise ValueError(f"{REGISTRY} at {commit} is not JSON") + return document -def base_registry(repository: Path) -> dict[str, Any] | None: +def base_registry(repository: Path) -> Any: """The registry at :func:`base_commit`, or ``None`` when the base has no registry yet. Raises ``LookupError`` when no base resolves.""" return registry_at(repository, base_commit(repository)) -def history_problems(repository: Path, base: str) -> list[str]: - """Append-only problems between every pair of consecutive first-parent - commits from ``base`` to ``HEAD``, so a push of several commits cannot - hide a change that a later commit undoes.""" +def pull_request_head() -> str | None: + """The pull request's head commit, on a ``pull_request`` event.""" + if os.environ.get("GITHUB_EVENT_NAME") != "pull_request": + return None + event = os.environ.get("GITHUB_EVENT_PATH") + if not event or not Path(event).is_file(): + return None + try: + payload = json.loads(Path(event).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + head = (payload.get("pull_request") or {}).get("head") or {} + sha = head.get("sha") if isinstance(head, Mapping) else None + return sha if isinstance(sha, str) and SHA_RX.fullmatch(sha) else None + + +def history_problems( + repository: Path, + base: str, + head: str = "HEAD", + *, + full: bool = True, + scored_runs: Mapping[str, tuple[str, str]] = SCORED_RUNS, + import_gates: frozenset[str] = HISTORICAL_IMPORT_GATES, +) -> list[str]: + """Problems across every first-parent commit from ``base`` to + ``head``, so a run of commits cannot hide a change a later commit + undoes. + + With ``full``, each snapshot is validated against its own commit's + tree. A snapshot that fails is reported and is not used as the next + one's predecessor. Each valid snapshot must append only to the last + valid one; a newly created registry is admitted against an empty + registry. Without ``full``, as for a pull request's own commits, only + the append-only rule is checked.""" listed = _git( - repository, "rev-list", "--first-parent", "--reverse", f"{base}..HEAD" + repository, + "rev-list", + "--first-parent", + "--reverse", + f"{base}..{head}", ) if listed.returncode != 0: - raise LookupError(f"cannot list commits from {base} to HEAD") + raise LookupError(f"cannot list commits from {base} to {head}") problems: list[str] = [] + try: + previous = registry_at(repository, base) + except ValueError as error: + problems.append(f"{base[:12]}: {error}") + previous = None + if not isinstance(previous, Mapping): + previous = None previous_commit = base - previous = registry_at(repository, base) for commit in listed.stdout.split(): - current = registry_at(repository, commit) - if current is not None: - shape = validate(current, vocabulary=current.get("sources", {})) + label = commit[:12] + try: + current = registry_at(repository, commit) + except ValueError as error: + problems.append(f"{label}: {error}") + continue + if current is None: + if previous is not None: + problems.append(f"{label}: the registry was deleted") + continue + if not isinstance(current, Mapping): + problems.append(f"{label}: registry: not a mapping") + continue + if full: + sources = current.get("sources") + shape = validate( + current, + repository, + vocabulary=sources if isinstance(sources, Mapping) else {}, + commit=commit, + scored_runs=scored_runs, + import_gates=import_gates, + ) if shape: - problems.extend(f"{commit[:12]}: {p}" for p in shape) - elif previous is not None: - problems.extend( - f"{previous_commit[:12]}..{commit[:12]}: {p}" - for p in append_only_problems(previous, current) - ) - elif previous is not None: - problems.append(f"{commit[:12]}: the registry was deleted") + problems.extend(f"{label}: {p}" for p in shape) + continue + before = previous if previous is not None else empty_like(current) + changes = append_only_problems(before, current) + if changes: + problems.extend( + f"{previous_commit[:12]}..{label}: {p}" for p in changes + ) + continue previous_commit, previous = commit, current return problems diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py index 247502c63..ecad1e9e4 100644 --- a/tests/test_track_b_holdout_registry.py +++ b/tests/test_track_b_holdout_registry.py @@ -11,6 +11,7 @@ from __future__ import annotations import copy +import hashlib import json import subprocess from pathlib import Path @@ -176,8 +177,30 @@ def make_registry(entries: list[dict], complete=False) -> dict: } -def validate(current, repository=None): - return registry.validate(current, repository, vocabulary=VOCABULARY) +#: The invented scored runs and historical-import gates, in place of the +#: module's controlled lists. Without a repository no pin is checked. +SCORED_RUNS = { + run_path(f"s{index}"): (f"gate_{index}", "0" * 64) for index in range(12) +} +IMPORT_GATES = frozenset(f"gate_{index}" for index in range(12)) +#: The legal transitions, specified independently of the module. +EXPECTED_TRANSITIONS = { + "candidate": {"frozen", "withdrawn"}, + "frozen": {"used", "withdrawn"}, + "used": set(), + "withdrawn": set(), +} + + +def validate(current, repository=None, scored_runs=None, commit="HEAD"): + return registry.validate( + current, + repository, + vocabulary=VOCABULARY, + commit=commit, + scored_runs=SCORED_RUNS if scored_runs is None else scored_runs, + import_gates=IMPORT_GATES, + ) entry_specs = st.lists( @@ -556,8 +579,9 @@ def test_repeated_ids_and_shared_gate_identifiers_are_refused(): [ ({"equals": "gate_other"}, "does not identify this gate"), ({"field": "title"}, "not an identity field"), - ({"path": "docs/design/registry.json"}, "is not runs/"), - ({"path": "../outside.json"}, "is not runs/"), + ({"path": "docs/design/registry.json"}, "not a listed scored run"), + ({"path": "../outside.json"}, "not a listed scored run"), + ({"path": run_path("s1")}, "'gate_1''s scored run"), ], ) def test_scored_evidence_must_be_a_run_artifact_naming_this_gate( @@ -571,6 +595,44 @@ def test_scored_evidence_must_be_a_run_artifact_naming_this_gate( assert any("needs scored_evidence" in p for p in problems) +def test_the_transitions_are_as_specified(): + assert { + status: set(following) + for status, following in registry.TRANSITIONS.items() + } == EXPECTED_TRANSITIONS + + +@pytest.mark.parametrize( + "statuses", + [ + ["candidate", "frozen", "used", "withdrawn"], + ["candidate", "withdrawn", "frozen"], + ["candidate", "frozen", "used", "frozen"], + ["candidate", "frozen", "frozen"], + ], +) +def test_terminal_and_repeated_statuses_are_refused(statuses): + entry = make_entry(0, ["s_a:2020"]) + entry["history"] = [ + {"date": "2026-10-21", "status": s, "note": "n"} for s in statuses + ] + entry["status"] = statuses[-1] + entry["scored_evidence"] = [ + {"path": run_path("s0"), "field": "gate", "equals": "gate_0"} + ] + assert any( + "not a legal transition" in p for p in validate(make_registry([entry])) + ) + + +def test_only_listed_gates_may_be_imported(): + entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + entry["gate"] = "gate_99" + entry["gate_identifiers"] = ["gate_99"] + problems = validate(make_registry([entry])) + assert any("scored before the registry existed" in p for p in problems) + + def test_a_review_must_name_an_earlier_entry(): entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_c:2025"])] entries[0]["overlap_review"].append(review("H002")) @@ -639,7 +701,7 @@ def test_any_json_in_any_field_is_reported_not_raised(key, value, append): assert isinstance(registry.append_only_problems(base, current), list) -# Committed trees and base commits, through git ---------------------------- +# Committed trees, scored runs and base commits, through git ---------------- def git(repository: Path, *args: str) -> str: @@ -664,31 +726,42 @@ def init_repository(tmp_path: Path) -> Path: "gates:\n gate_0:\n id: zero_gate\n" " registration_id: reg-zero\n", ) - git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", "start") + commit_all(repository, "start") return repository def write(repository: Path, relative: str, document) -> None: path = repository / relative path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(document, bytes): + path.write_bytes(document) + return text = document if isinstance(document, str) else json.dumps(document) path.write_text(text, encoding="utf-8") def commit_all(repository: Path, message: str) -> str: git(repository, "add", "-A") - git(repository, "commit", "-q", "-m", message) + git(repository, "commit", "-q", "--allow-empty", "-m", message) return git(repository, "rev-parse", "HEAD").strip() +def pin(document, gate: str = "gate_0") -> dict: + """The scored-run list for one artifact holding ``document``.""" + text = json.dumps(document).encode("utf-8") + return {run_path("s0"): (gate, hashlib.sha256(text).hexdigest())} + + @pytest.fixture def no_ci_env(monkeypatch): for name in ("GITHUB_BASE_REF", "GITHUB_EVENT_NAME", "GITHUB_EVENT_PATH"): monkeypatch.delenv(name, raising=False) -def scored_registry() -> dict: +SCORE = {"gate": "gate_0"} + + +def imported(document=SCORE) -> dict: entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) entry["gate_identifiers"] = ["gate_0", "zero_gate", "reg-zero"] return make_registry([entry]) @@ -696,93 +769,248 @@ def scored_registry() -> dict: def test_scored_evidence_is_read_from_the_committed_tree(tmp_path, no_ci_env): repository = init_repository(tmp_path) - current = scored_registry() + current = imported() evidence = current["entries"][0]["scored_evidence"][0] - write(repository, evidence["path"], {"gate": "gate_0"}) + runs = pin(SCORE) + write(repository, evidence["path"], SCORE) assert any( - "not a committed file" in p for p in validate(current, repository) + "not a committed file" in p + for p in validate(current, repository, runs) ) git(repository, "add", "-A") assert any( - "not a committed file" in p for p in validate(current, repository) + "not a committed file" in p + for p in validate(current, repository, runs) ) git(repository, "commit", "-q", "-m", "score") - assert validate(current, repository) == [] + assert validate(current, repository, runs) == [] write(repository, evidence["path"], {"gate": "gate_9"}) - assert validate(current, repository) == [] + assert validate(current, repository, runs) == [] for field, equals in ( ("registration_id", "reg-zero"), ("gate", "zero_gate"), ): evidence.update(field=field, equals=equals) - assert any("does not have" in p for p in validate(current, repository)) + assert any( + "does not have" in p for p in validate(current, repository, runs) + ) + + +def test_a_scored_run_must_keep_its_pinned_bytes(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + current = imported() + path = current["entries"][0]["scored_evidence"][0]["path"] + write(repository, path, {"gate": "gate_0", "floor": True}) + commit_all(repository, "a floor, not the scored run") + problems = validate(current, repository, pin(SCORE)) + assert any("differs from the listed scored run" in p for p in problems) def test_identifiers_must_be_authoritative(tmp_path, no_ci_env): repository = init_repository(tmp_path) - current = scored_registry() + current = imported() entry = current["entries"][0] - write(repository, entry["scored_evidence"][0]["path"], {"title": "x"}) + document = {"gate": "x"} + write(repository, entry["scored_evidence"][0]["path"], document) commit_all(repository, "score") entry["gate_identifiers"].append("x") entry["scored_evidence"][0].update(field="gate", equals="x") assert any( "does not identify this gate" in p - for p in validate(current, repository) + for p in validate(current, repository, pin(document)) ) entry["gate"] = "gate_unknown" entry["gate_identifiers"] = ["gate_unknown"] entry["scored_evidence"][0]["equals"] = "gate_unknown" + runs = pin(document, "gate_unknown") assert any( - "neither a gates.yaml gate" in p for p in validate(current, repository) + "neither a gates.yaml gate" in p + for p in validate(current, repository, runs) ) def test_a_symlinked_or_non_json_artifact_is_refused(tmp_path, no_ci_env): repository = init_repository(tmp_path) - current = scored_registry() - evidence = current["entries"][0]["scored_evidence"][0] + current = imported() + path = current["entries"][0]["scored_evidence"][0]["path"] outside = tmp_path / "outside.json" - outside.write_text(json.dumps({"gate": "gate_0"}), encoding="utf-8") - link = repository / evidence["path"] + outside.write_text(json.dumps(SCORE), encoding="utf-8") + link = repository / path link.parent.mkdir(parents=True) link.symlink_to(outside) commit_all(repository, "link") assert any( - "not a committed file" in p for p in validate(current, repository) + "not a committed file" in p + for p in validate(current, repository, pin(SCORE)) ) link.unlink() - write(repository, evidence["path"], "not json") - commit_all(repository, "text") - assert any("not JSON" in p for p in validate(current, repository)) + write(repository, path, b"\xff\xfe not json") + commit_all(repository, "bytes") + runs = {path: ("gate_0", hashlib.sha256(b"\xff\xfe not json").hexdigest())} + assert any( + "does not have" in p for p in validate(current, repository, runs) + ) -def test_imports_need_evidence_older_than_the_registry(tmp_path, no_ci_env): - repository = init_repository(tmp_path) - current = scored_registry() - evidence = current["entries"][0]["scored_evidence"][0] - write(repository, evidence["path"], {"gate": "gate_0"}) - commit_all(repository, "score before the registry") +def registered(statuses) -> dict: + """A registration of gate_0 at ``statuses``, citing s0 once used.""" + entry = make_entry(0, ["s_a:2020"]) + entry["gate_identifiers"] = ["gate_0", "zero_gate", "reg-zero"] + entry["history"] = [ + {"date": f"2026-10-{21 + step:02d}", "status": s, "note": "n"} + for step, s in enumerate(statuses) + ] + entry["status"] = statuses[-1] + if statuses[-1] == "used": + entry["scored_evidence"] = [ + {"path": run_path("s0"), "field": "gate", "equals": "gate_0"} + ] + return make_registry([entry]) + + +def commit_registry(repository: Path, current: dict, message: str) -> str: write(repository, registry.REGISTRY, current) - commit_all(repository, "registry") - assert validate(current, repository) == [] - late = make_entry(1, ["s_c:2099"], ("historical_import", "used")) - late["gate"] = "gate_0" - late["gate_identifiers"] = ["gate_0"] - late["scored_evidence"][0].update(path=run_path("late"), equals="gate_0") - write(repository, late["scored_evidence"][0]["path"], {"gate": "gate_0"}) - commit_all(repository, "score after the registry") - current["entries"].append(late) - assert any( - "after the registry existed" in p - for p in validate(current, repository) + return commit_all(repository, message) + + +@pytest.mark.parametrize("registered_first", [True, False]) +def test_a_registration_must_precede_its_scored_run( + tmp_path, no_ci_env, registered_first +): + repository = init_repository(tmp_path) + if not registered_first: + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score first") + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + if registered_first: + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + if registered_first: + assert problems == [] + else: + assert any("already committed when" in p for p in problems) + + +def test_a_used_registration_must_already_be_registered(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + problems = validate(used, repository, pin(SCORE)) + assert any("before it is registered" in p for p in problems) + + +def test_each_snapshot_is_validated_against_its_own_tree(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = git(repository, "rev-parse", "HEAD").strip() + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used before its score is committed") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "the score, one commit late") + runs = pin(SCORE) + assert validate(used, repository, runs) == [] + problems = registry.history_problems( + repository, base, scored_runs=runs, import_gates=IMPORT_GATES ) + assert any("not a committed file" in p for p in problems) -def test_review_citations_must_name_committed_lines(tmp_path, no_ci_env): +def test_a_new_registry_is_admitted_against_an_empty_one(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = git(repository, "rev-parse", "HEAD").strip() + commit_registry( + repository, registered(["candidate", "frozen"]), "arrives frozen" + ) + problems = registry.history_problems( + repository, base, import_gates=IMPORT_GATES + ) + assert any("lone candidate" in p for p in problems) + + +def test_a_clean_history_has_no_problems(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = git(repository, "rev-parse", "HEAD").strip() + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + commit_registry( + repository, registered(["candidate", "frozen", "used"]), "used" + ) + assert ( + registry.history_problems( + repository, + base, + scored_runs=pin(SCORE), + import_gates=IMPORT_GATES, + ) + == [] + ) + + +@pytest.mark.parametrize("undo", ["append_then_remove", "edit_then_restore"]) +@pytest.mark.parametrize("full", [True, False]) +def test_a_run_of_commits_cannot_hide_a_change_it_undoes( + tmp_path, no_ci_env, undo, full +): + repository = init_repository(tmp_path) + first = registered(["candidate"]) + base = commit_registry(repository, first, "registry") + changed = copy.deepcopy(first) + if undo == "append_then_remove": + changed["entries"].append(make_entry(1, ["s_c:2099"])) + else: + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "change") + commit_registry(repository, first, "undo") + assert registry.append_only_problems(first, first) == [] + assert registry.history_problems( + repository, base, full=full, import_gates=IMPORT_GATES + ) + + +def test_a_pull_request_head_is_read_from_its_event(tmp_path, monkeypatch): + event = tmp_path / "event.json" + sha = "a" * 40 + event.write_text( + json.dumps({"pull_request": {"head": {"sha": sha}}}), encoding="utf-8" + ) + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + assert registry.pull_request_head() == sha + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + assert registry.pull_request_head() is None + + +def test_malformed_snapshots_are_reported_not_raised(tmp_path, no_ci_env): + repository = init_repository(tmp_path) + base = commit_registry(repository, registered(["candidate"]), "registry") + for bad in ("not json", [], {"entries": None}): + write( + repository, + registry.REGISTRY, + bad if isinstance(bad, str) else json.dumps(bad), + ) + commit_all(repository, "a malformed snapshot") + commit_registry(repository, registered(["candidate"]), "repaired") + problems = registry.history_problems( + repository, base, import_gates=IMPORT_GATES + ) + assert len(problems) >= 3 + assert registry.append_only_problems(registered(["candidate"]), None) + + +def test_review_citations_must_name_committed_text_lines(tmp_path, no_ci_env): repository = init_repository(tmp_path) write(repository, "docs/note.md", "one\ntwo\nthree\n") - commit_all(repository, "note") + write(repository, "docs/binary.md", b"\xff\xfe\x00\x01") + commit_all(repository, "notes") entries = [make_entry(0, ["s_a:2020"]), make_entry(1, ["s_b:2025"])] entries[1]["overlap_review"].append(review("H001")) current = make_registry(entries) @@ -791,6 +1019,9 @@ def test_review_citations_must_name_committed_lines(tmp_path, no_ci_env): ("docs/note.md:2-3", True), ("docs/note.md:4", False), ("docs/missing.md:1", False), + ("docs/binary.md:1", False), + ("docs/note.md:12345678", False), + ("docs/../docs/note.md:2", False), ): entries[1]["overlap_review"][0]["evidence"] = [citation] assert (validate(current, repository) == []) == ok, citation @@ -803,11 +1034,10 @@ def test_complete_needs_its_committed_record(tmp_path, no_ci_env): write(repository, current["completion_record"], {}) commit_all(repository, "inventory") assert validate(current, repository) == [] - - -def commit_registry(repository: Path, current: dict, message: str) -> str: - write(repository, registry.REGISTRY, current) - return commit_all(repository, message) + for path in ("docs/a\x00b.json", "docs/../docs/design/inventory.json"): + current["completion_record"] = path + problems = validate(current, repository) + assert any("is not a path" in p for p in problems), path def test_a_pull_request_compares_with_its_merge_base(tmp_path, no_ci_env): @@ -817,7 +1047,7 @@ def test_a_pull_request_compares_with_its_merge_base(tmp_path, no_ci_env): commit_all(repository, "feature work") assert registry.base_registry(repository) is None git(repository, "checkout", "-q", "master") - first = build([(["s_a:2020"], ("historical_import", "used"))]) + first = registered(["candidate"]) commit_registry(repository, first, "registry") git(repository, "checkout", "-q", "-b", "feature-2") second = copy.deepcopy(first) @@ -828,7 +1058,7 @@ def test_a_pull_request_compares_with_its_merge_base(tmp_path, no_ci_env): def test_the_base_branch_compares_with_its_parent(tmp_path, no_ci_env): repository = init_repository(tmp_path) - first = build([(["s_a:2020"], ("historical_import", "used"))]) + first = registered(["candidate"]) commit_registry(repository, first, "registry") emptied = copy.deepcopy(first) emptied["entries"] = [] @@ -842,7 +1072,7 @@ def test_a_push_compares_with_its_before_commit( tmp_path, no_ci_env, monkeypatch ): repository = init_repository(tmp_path) - first = build([(["s_a:2020"], ("historical_import", "used"))]) + first = registered(["candidate"]) before = commit_registry(repository, first, "registry") middle = copy.deepcopy(first) middle["entries"].append(make_entry(1, ["s_c:2099"])) @@ -862,35 +1092,6 @@ def test_a_push_compares_with_its_before_commit( assert registry.base_registry(repository) == middle -@pytest.mark.parametrize("undo", ["append_then_remove", "edit_then_restore"]) -def test_a_push_cannot_hide_a_change_a_later_commit_undoes( - tmp_path, no_ci_env, undo -): - repository = init_repository(tmp_path) - first = build([(["s_a:2020"], ("historical_import", "used"))]) - base = commit_registry(repository, first, "registry") - changed = copy.deepcopy(first) - if undo == "append_then_remove": - changed["entries"].append(make_entry(1, ["s_c:2099"])) - commit_registry(repository, changed, "append") - else: - changed["entries"][0]["unit_keys"] = ["s_c:2199"] - commit_registry(repository, changed, "edit a key") - commit_registry(repository, first, "undo") - assert registry.append_only_problems(first, first) == [] - assert registry.history_problems(repository, base) - - -def test_a_clean_history_has_no_problems(tmp_path, no_ci_env): - repository = init_repository(tmp_path) - first = build([(["s_a:2020"], ("historical_import", "used"))]) - base = commit_registry(repository, first, "registry") - second = copy.deepcopy(first) - second["entries"].append(make_entry(1, ["s_c:2099"])) - commit_registry(repository, second, "append a candidate") - assert registry.history_problems(repository, base) == [] - - def test_no_base_raises_rather_than_passing(tmp_path, monkeypatch): monkeypatch.delenv("GITHUB_EVENT_NAME", raising=False) monkeypatch.setenv("GITHUB_BASE_REF", "no-such-branch") diff --git a/tests/test_track_b_holdout_registry_committed.py b/tests/test_track_b_holdout_registry_committed.py index 6c7a6133e..a2322ba58 100644 --- a/tests/test_track_b_holdout_registry_committed.py +++ b/tests/test_track_b_holdout_registry_committed.py @@ -11,6 +11,7 @@ from __future__ import annotations +import hashlib import os from pathlib import Path @@ -65,6 +66,26 @@ "mermin_smith_anchors": "literature", } EXPECTED_OUTSIDE_GATES_YAML = {"B2": ("B2", "6252388925")} +#: Every listed scored run and its gate, specified here independently. +EXPECTED_SCORED_RUN_GATES = { + "runs/gate_m6_candidate1_v1.json": "gate_m6", + "runs/gate_m6_candidate2_v1.json": "gate_m6", + "runs/gate_m6_candidate3_v1.json": "gate_m6", + "runs/track_b_b2_score.json": "B2", + "runs/epuf_fill_gate_test_v1.json": "gate_epuf_fill", + "runs/gate_m4_hazard_v1.json": "gate_m4", + "runs/gate_w1_candidate1_v1.json": "gate_w1", + "runs/gate_w1_candidate2_v1.json": "gate_w1", + "runs/gate_w1_candidate3_v1.json": "gate_w1", + "runs/gate_w1_candidate4_v1.json": "gate_w1", +} +EXPECTED_IMPORT_GATES = { + "gate_m6", + "B2", + "gate_epuf_fill", + "gate_m4", + "gate_w1", +} def committed() -> dict: @@ -91,9 +112,31 @@ def resolved_base(): def test_the_committed_registry_only_appends_to_its_base(): base_commit = resolved_base() base = registry.registry_at(ROOT, base_commit) - if base is not None: + current = committed() + if base is None: + assert ( + registry.append_only_problems( + registry.empty_like(current), current + ) + == [] + ) + else: assert registry.validate(base, vocabulary=base["sources"]) == [] - assert registry.append_only_problems(base, committed()) == [] + assert registry.append_only_problems(base, current) == [] + + +def test_a_pull_requests_own_commits_append_only(): + head = registry.pull_request_head() + if head is None: + pytest.skip("not a pull_request event") + base_commit = resolved_base() + if registry.registry_at(ROOT, base_commit) is None: + pytest.skip( + "the registry is new in this pull request, which is merged with " + "a merge commit; its endpoint is admitted against an empty " + "registry above" + ) + assert registry.history_problems(ROOT, base_commit, head, full=False) == [] def test_every_first_parent_commit_since_the_base_appends_only(): @@ -144,6 +187,16 @@ def test_the_vocabulary_and_registrations_are_as_specified(): } +def test_the_scored_runs_and_import_gates_are_as_specified(): + assert { + path: gate for path, (gate, _) in registry.SCORED_RUNS.items() + } == EXPECTED_SCORED_RUN_GATES + assert set(registry.HISTORICAL_IMPORT_GATES) == EXPECTED_IMPORT_GATES + for path, (_, digest) in registry.SCORED_RUNS.items(): + content = (ROOT / path).read_bytes() + assert hashlib.sha256(content).hexdigest() == digest, path + + def test_every_used_entry_cites_committed_run_artifacts(): for entry in committed()["entries"]: if entry["status"] == "used": diff --git a/tests/tier_counts.json b/tests/tier_counts.json index dc36b664c..55de32e2a 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,8 +1,8 @@ { "schema_version": 1, "counts": { - "unit": 10032, - "artifact": 4261, + "unit": 10048, + "artifact": 4263, "integration_psid": 1450, "reproduction_legacy": 520, "oracle_policyengine": 308 From 5f7d995d72dfc6b5a813c6199c8052028685b233 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 09:08:08 -0400 Subject: [PATCH 5/9] Holdout registry: answer review round 4 (two medium, two low) 1. Moving or removing a score bypassed preregistration (medium). The timing check now asks whether the scored run's exact bytes were committed, at any path, in the history of the commit that first records the entry on the first-parent chain (git hash-object, then git log -m --find-object). A score committed elsewhere and moved, or committed, deleted and restored, or committed on the base branch while the registration waited on a side branch, is refused. When git cannot answer, the check fails closed. 2. Historical-import permission could be reused for unscored units (medium). HISTORICAL_IMPORTS replaces the list of gates: each authorization binds one entry id to its gate, unit keys and scored runs, so an import under another id, with other units or with other runs is refused. A correction found by B0.2's inventory is a new authorized entry in a reviewed code change. 3. The first-parent registration rule had no regression test (low). The side-branch scenario now kills the --topo-order mutant. 4. Malformed inputs still raised (low). Entries without string ids are reported as malformed before any diagnostic reads them; JSON and YAML parser limits (a 5,000-digit integer) and NUL arguments to git return problems, or registry_at's documented ValueError and LookupError. Rules 6-8 in the registry now describe these mechanisms, and rule 7 states that a pull request touching the registry is merged with a merge commit, never rebased (the review's nonblocking point 5). Known and left to B0.2's inventory (nonblocking in the review): legacy gates gate_1, gate_2, gate_2b and gate_2c have committed scored outputs that are not yet imported; the registry stays incomplete until then. The reviewer's own counterexample scripts, replayed on this code: every import and timing bypass is refused, and the valid order still passes. Mutations: 64 of 64 killed, including the reviewer's --topo-order survivor and new checks on the authorization, the any-path timing search, fail-closed timing, malformed ids, parser limits and NUL arguments. Tests: 122 passed, 2 skipped (with the birth-evidence test). Tier counts: unit 10,059 (+11), artifact 4,263. Co-Authored-By: Claude Opus 5.5 --- docs/design/track_b_holdout_registry.json | 6 +- .../track_b_holdout_registry_mutations.json | 143 +++++++++++- .../track_b/holdout_registry.py | 217 ++++++++++++----- tests/test_track_b_holdout_registry.py | 221 ++++++++++++++++-- ...test_track_b_holdout_registry_committed.py | 36 ++- tests/tier_counts.json | 2 +- 6 files changed, 530 insertions(+), 95 deletions(-) diff --git a/docs/design/track_b_holdout_registry.json b/docs/design/track_b_holdout_registry.json index bac6d505b..bf4c70245 100644 --- a/docs/design/track_b_holdout_registry.json +++ b/docs/design/track_b_holdout_registry.json @@ -9,9 +9,9 @@ "Two unit keys share a unit when their source is the same and their years intersect; scope segments describe the unit and never narrow the overlap. Live entries that share a unit, or whose relation is unresolved, list each other in disclosed_overlaps, and both gates' registrations name the overlap. No unit is held out by two gates silently.", "The overlap rule cannot see two sources that share underlying data, or two editions of a publication that report the same data years. So an entry reviews every earlier live entry of the same source family that it neither shares a unit with nor discloses. A review records the outcome 'disjoint', a substantive reason and citations (path:line or https URLs); an unresolved relation is disclosed instead.", "Reference years are recorded separately from interview or publication availability (design section 5.2, element 10): a unit key's year is what its source declares (for PSID, the interview wave; for EPUF, the file's vintage year), and each entry's units state the reference or event years.", - "An entry whose gate has scored has status 'used'. Its scored_evidence cites scored-run artifacts listed in the module's SCORED_RUNS, which pins each artifact's gate and the SHA-256 of its bytes; an artifact joins that list, as a reviewed code change, when its gate scores. Each citation also names an identity field (gate, gate.name or registration_id) whose value identifies the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. A registration's scored run must not have been committed when the entry was registered. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", - "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused. These rules hold against the base commit, between every pair of consecutive first-parent commits on the base branch (each snapshot also validated against its own commit's tree), and between consecutive commits of a pull request's branch. A newly created registry is admitted against an empty one.", - "A historical import records a gate that scored before this registry existed. It starts as used, and its gate must be in the module's HISTORICAL_IMPORT_GATES, which B0.2's mechanical inventory may extend by a reviewed code change. Imports are allowed only while the registry is incomplete.", + "An entry whose gate has scored has status 'used'. Its scored_evidence cites scored-run artifacts listed in the module's SCORED_RUNS, which pins each artifact's gate and the SHA-256 of its bytes; an artifact joins that list, as a reviewed code change, when its gate scores. Each citation also names an identity field (gate, gate.name or registration_id) whose value identifies the gate in gates.yaml or in the module's REGISTRATIONS_OUTSIDE_GATES_YAML. A registration's scored-run bytes must not have been committed, at any path and whether or not they were later moved or deleted, in the history of the commit that first records the entry on the base branch's first-parent chain. Its units are exposure for every later gate and never again an unused holdout (design section 5.2).", + "Entries are only appended. An entry's id, provenance, gate, lane, source, units and unit_keys never change; its gate identifiers, evidence, scored evidence, disclosures, reviews and history only extend. Ids are never reused. These rules hold against the base commit, between every pair of consecutive first-parent commits on the base branch (each snapshot also validated against its own commit's tree), and between consecutive commits of a pull request's branch. A newly created registry is admitted against an empty one. A pull request that creates or changes the registry is merged with a merge commit, never rebased, so its draft commits do not enter the base branch's first-parent history.", + "A historical import records a gate that scored before this registry existed. It starts as used, and its id, gate, unit keys and scored runs must match an authorization in the module's HISTORICAL_IMPORTS. B0.2's mechanical inventory adds an import it finds, or a correction to an existing one, as a new authorized entry in a reviewed code change. Imports are allowed only while the registry is incomplete.", "The registry is incomplete until completion_record, B0.2's mechanical exposure inventory, exists. It starts with the prior uses verified below; the inventory adds the rest before any B0.2 gate record freezes." ], "status_values": { diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json index 693c8f70e..069638a4c 100644 --- a/docs/design/track_b_holdout_registry_mutations.json +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -4,7 +4,7 @@ "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", "mutation_method": "One deliberate production-line change per check; both registry test modules run in full and are expected to fail; original bytes restored before the next check. No mutated implementation is committed.", "restored_source_sha256": { - "src/populace_dynamics/track_b/holdout_registry.py": "aae22b1de559901fd563897bf6f7ce7296f1cc00caa46e32163cbe89ab6997f6" + "src/populace_dynamics/track_b/holdout_registry.py": "0d82ef49bae6458c00e7c5386779ddad521b3eab60f63105600fb5c0f5bfec1b" }, "checks": [ { @@ -64,7 +64,6 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", "tests/test_track_b_holdout_registry_committed.py::test_the_seeded_disclosures_are_as_specified" @@ -91,13 +90,60 @@ ] }, { - "mutation": "any gate may be imported", - "old": "and entry[\"gate\"] not in import_gates", - "new": "and False", + "mutation": "any id may be imported", + "old": "return [f\"{name}: is not an authorized historical import\"]", + "new": "return []", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_authorization_admits_one_import_only", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_rename-not an authorized historical import]" + ] + }, + { + "mutation": "imports need no authorization", + "old": "problems.extend(_import_problems(entry, imports))", + "new": "pass", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_only_listed_gates_may_be_imported" + "tests/test_track_b_holdout_registry.py::test_an_authorization_admits_one_import_only", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_runs-differs from its authorized runs]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_units-unit keys differ]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_other_units-unit keys differ]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_regate-is authorized for 'gate_0']", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_rename-not an authorized historical import]" + ] + }, + { + "mutation": "imports may change gate", + "old": "if entry[\"gate\"] != authorized[\"gate\"]:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_regate-is authorized for 'gate_0']" + ] + }, + { + "mutation": "imports may change units", + "old": "if sorted(entry[\"unit_keys\"]) != sorted(authorized[\"unit_keys\"]):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_units-unit keys differ]", + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_other_units-unit keys differ]" + ] + }, + { + "mutation": "imports may cite other runs", + "old": "if cited != sorted(authorized[\"scored_runs\"]):", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_imports_must_match_their_authorization[_more_runs-differs from its authorized runs]" ] }, { @@ -196,12 +242,90 @@ }, { "mutation": "registration may follow its score", - "old": "if earlier is not None and (", - "new": "if False and (", + "old": "elif earlier:", + "new": "elif False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[deleted_and_restored]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[moved]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[same_commit]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]" + ] + }, + { + "mutation": "an unanswerable timing check passes", + "old": "if earlier is None:", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_unanswerable_timing_check_fails_closed" + ] + }, + { + "mutation": "scores never found in history", + "old": "return bool(found.stdout.strip())", + "new": "return False", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[deleted_and_restored]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[moved]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[same_commit]", + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]" + ] + }, + { + "mutation": "registration order ignores the first parent", + "old": "\"log\",\n \"--first-parent\",", + "new": "\"log\",\n \"--topo-order\",", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]" + ] + }, + { + "mutation": "entries without ids reach diagnostics", + "old": "isinstance(entry, Mapping) and isinstance(entry.get(\"id\"), str)", + "new": "isinstance(entry, Mapping)", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "JSON parser limits raise", + "old": "except (ValueError, RecursionError):\n return None", + "new": "except json.JSONDecodeError:\n return None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "YAML parser limits raise", + "old": "except (yaml.YAMLError, ValueError, RecursionError):", + "new": "except yaml.YAMLError:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" + ] + }, + { + "mutation": "NUL arguments raise", + "old": "except ValueError as error:\n return subprocess.CompletedProcess(command, 128, \"\", str(error))", + "new": "except OSError as error:\n return subprocess.CompletedProcess(command, 128, \"\", str(error))", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]" + "tests/test_track_b_holdout_registry.py::test_malformed_predecessors_and_parser_limits_are_reported" ] }, { @@ -472,7 +596,6 @@ "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]", - "tests/test_track_b_holdout_registry.py::test_any_json_in_any_field_is_reported_not_raised", "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused", "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" ] diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py index 4a715dc1c..f864a71ca 100644 --- a/src/populace_dynamics/track_b/holdout_registry.py +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -25,10 +25,11 @@ - unique entry ids, which are never removed, reordered or reused; - histories with legal transitions only. A registration starts as a lone ``candidate`` in a commit of its own, so it reaches the base branch - before its gate freezes, and its scored run must not yet be committed - when it is registered. A historical import is a gate that scored before - the registry existed: it starts as ``used``, and its gate must be in - :data:`HISTORICAL_IMPORT_GATES`; + before its gate freezes, and its scored run's bytes must not yet have + been committed, at any path, when it is registered. A historical import + is a gate that scored before the registry existed: it starts as + ``used``, and its id, gate, unit keys and scored runs must match an + authorization in :data:`HISTORICAL_IMPORTS`; - a ``used`` entry cites scored-run artifacts listed in :data:`SCORED_RUNS`, which pins each one's gate and the SHA-256 of its bytes, read from the committed tree; each citation's identity field @@ -210,11 +211,74 @@ ), } ) -#: The gates that scored before the registry existed and may enter it as -#: historical imports. B0.2's mechanical inventory adds any it finds, as a -#: reviewed code change; no other gate can import itself. -HISTORICAL_IMPORT_GATES = frozenset( - {"gate_m6", "B2", "gate_epuf_fill", "gate_m4", "gate_w1"} +#: The historical imports this module authorizes: gates that scored before +#: the registry existed. Each binds one entry id to its gate, its unit keys +#: and its scored runs, so the exception to preregistration covers exactly +#: the exposure reviewed here. B0.2's mechanical inventory adds any import +#: it finds, or a correction to an existing one, as a new authorized entry +#: in a reviewed code change; nothing else can enter as an import. +HISTORICAL_IMPORTS: Mapping[str, Mapping[str, Any]] = MappingProxyType( + { + "H001": MappingProxyType( + { + "gate": "gate_m6", + "unit_keys": ("psid:2015-2023:held_out_window",), + "scored_runs": ( + "runs/gate_m6_candidate1_v1.json", + "runs/gate_m6_candidate2_v1.json", + "runs/gate_m6_candidate3_v1.json", + ), + } + ), + "H002": MappingProxyType( + { + "gate": "B2", + "unit_keys": ("psid:2013:fam2013er", "psid:2015:fam2015er"), + "scored_runs": ("runs/track_b_b2_score.json",), + } + ), + "H003": MappingProxyType( + { + "gate": "gate_epuf_fill", + "unit_keys": ("ssa_epuf:2006:test_partition",), + "scored_runs": ("runs/epuf_fill_gate_test_v1.json",), + } + ), + "H004": MappingProxyType( + { + "gate": "gate_m4", + "unit_keys": ( + "psid:1982:employment_status", + "psid:1983:employment_status", + "psid:1993-1997:employment_status", + "psid:1999-2023:employment_status", + "ssa_di_asr:2023", + ), + "scored_runs": ("runs/gate_m4_hazard_v1.json",), + } + ), + "H005": MappingProxyType( + { + "gate": "gate_w1", + "unit_keys": ( + "ssa_supplement:2023:6.b5.1", + "ssa_supplement:2023:6.b4", + "ssa_supplement:2023:6.b3", + "ssa_di_asr:2023:table_19", + "census_cps_tables:2023:hh-4", + "census_cps_tables:2023:ad-3", + "populace_us_frame:2024:certified", + "mermin_smith_anchors:undated", + ), + "scored_runs": ( + "runs/gate_w1_candidate1_v1.json", + "runs/gate_w1_candidate2_v1.json", + "runs/gate_w1_candidate3_v1.json", + "runs/gate_w1_candidate4_v1.json", + ), + } + ), + } ) STATUSES = ("candidate", "frozen", "used", "withdrawn") TRANSITIONS = { @@ -368,20 +432,29 @@ def lookup(document: Any, dotted: str) -> Any: def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: - return subprocess.run( - ["git", "-C", str(repository), *args], - capture_output=True, - text=True, - check=False, - ) + """Run git; an argument git cannot take (a NUL byte) fails like a git + error rather than raising.""" + command = ["git", "-C", str(repository), *args] + try: + return subprocess.run( + command, capture_output=True, text=True, check=False + ) + except ValueError as error: + return subprocess.CompletedProcess(command, 128, "", str(error)) -def _git_bytes(repository: Path, *args: str) -> bytes | None: - found = subprocess.run( - ["git", "-C", str(repository), *args], - capture_output=True, - check=False, - ) +def _git_bytes( + repository: Path, *args: str, stdin: bytes | None = None +) -> bytes | None: + try: + found = subprocess.run( + ["git", "-C", str(repository), *args], + input=stdin, + capture_output=True, + check=False, + ) + except ValueError: + return None return found.stdout if found.returncode == 0 else None @@ -400,7 +473,7 @@ def _parse_registry(text: str | None) -> Any: return None try: return json.loads(text) - except json.JSONDecodeError: + except (ValueError, RecursionError): return None @@ -451,7 +524,7 @@ def identifiers(self, gate: str) -> set[str]: text = self.text(GATES) try: document = yaml.safe_load(text) if text else {} - except yaml.YAMLError: + except (yaml.YAMLError, ValueError, RecursionError): document = {} gates = ( document.get("gates") if isinstance(document, dict) else None @@ -502,6 +575,28 @@ def registered_at(self, entry_id: str) -> str | None: self._registered.setdefault(name, commit) return self._registered.get(entry_id) + def committed_by(self, content: bytes, commit: str) -> bool | None: + """Whether a file with exactly ``content`` was committed, at any + path and whether or not it was later moved or deleted, in + ``commit`` or any of its ancestors. ``None`` when git cannot say.""" + name = _git_bytes( + self.repository, "hash-object", "--stdin", stdin=content + ) + if not name or not name.strip(): + return None + found = _git( + self.repository, + "log", + "-m", + "-1", + "--format=%H", + f"--find-object={name.decode().strip()}", + commit, + ) + if found.returncode != 0: + return None + return bool(found.stdout.strip()) + def _decode(content: bytes | None) -> str | None: if content is None: @@ -512,24 +607,42 @@ def _decode(content: bytes | None) -> str | None: return None +def _import_problems( + entry: Mapping[str, Any], imports: Mapping[str, Mapping[str, Any]] +) -> list[str]: + """Problems with a historical import against its authorization.""" + name = entry["id"] + authorized = imports.get(name) + if authorized is None: + return [f"{name}: is not an authorized historical import"] + problems = [] + if entry["gate"] != authorized["gate"]: + problems.append( + f"{name}: is authorized for {authorized['gate']!r}, not " + f"{entry['gate']!r}" + ) + if sorted(entry["unit_keys"]) != sorted(authorized["unit_keys"]): + problems.append(f"{name}: unit keys differ from its authorization") + cited = sorted(item["path"] for item in entry["scored_evidence"]) + if cited != sorted(authorized["scored_runs"]): + problems.append( + f"{name}: scored evidence differs from its authorized runs" + ) + return problems + + def _evidence_problems( entry: Mapping[str, Any], committed: Committed | None, scored_runs: Mapping[str, tuple[str, str]], - import_gates: frozenset[str], + imports: Mapping[str, Mapping[str, Any]], ) -> list[str]: """Problems with an entry's scored-evidence citations and, for a - historical import, with its gate.""" + historical import, with its authorization.""" name = entry["id"] problems: list[str] = [] - if ( - entry["provenance"] == "historical_import" - and entry["gate"] not in import_gates - ): - problems.append( - f"{name}: {entry['gate']!r} is not a gate that scored before the " - "registry existed" - ) + if entry["provenance"] == "historical_import": + problems.extend(_import_problems(entry, imports)) if not entry["scored_evidence"]: return problems allowed = set(entry["gate_identifiers"]) @@ -594,13 +707,16 @@ def _evidence_problems( ) continue if registered is not None: - earlier = committed.blob(path, registered) - if earlier is not None and ( - hashlib.sha256(earlier).hexdigest() == run[1] - ): + earlier = committed.committed_by(content, registered) + if earlier is None: problems.append( - f"{name}: its scored run {path!r} was already committed " - "when the entry was registered" + f"{name}: cannot tell whether its scored run {path!r} " + "was committed before it was registered" + ) + elif earlier: + problems.append( + f"{name}: its scored run {path!r} was already committed, " + "at some path, when the entry was registered" ) return problems @@ -700,7 +816,7 @@ def _entry_problems( sources: Mapping[str, Any], committed: Committed | None, scored_runs: Mapping[str, tuple[str, str]], - import_gates: frozenset[str], + imports: Mapping[str, Mapping[str, Any]], ) -> list[str]: """Problems inside one well-shaped entry.""" name = entry["id"] @@ -760,9 +876,7 @@ def _entry_problems( ) if entry["status"] == "used" and not entry["scored_evidence"]: problems.append(f"{name}: a used entry needs scored_evidence") - problems.extend( - _evidence_problems(entry, committed, scored_runs, import_gates) - ) + problems.extend(_evidence_problems(entry, committed, scored_runs, imports)) problems.extend(_review_problems(entry, committed)) return problems @@ -799,12 +913,12 @@ def validate( *, commit: str = "HEAD", scored_runs: Mapping[str, tuple[str, str]] = SCORED_RUNS, - import_gates: frozenset[str] = HISTORICAL_IMPORT_GATES, + imports: Mapping[str, Mapping[str, Any]] = HISTORICAL_IMPORTS, ) -> list[str]: """Every rule one registry must keep on its own. ``vocabulary`` is the controlled source list the registry's copy must equal; - ``scored_runs`` and ``import_gates`` are the controlled lists of - scored runs and historical-import gates. With ``repository``, scored + ``scored_runs`` and ``imports`` are the controlled lists of scored + runs and authorized historical imports. With ``repository``, scored evidence, citations and the completion record are checked against its ``commit``.""" if not isinstance(registry, Mapping): @@ -867,9 +981,7 @@ def validate( ) for entry in entries: problems.extend( - _entry_problems( - entry, vocabulary, committed, scored_runs, import_gates - ) + _entry_problems(entry, vocabulary, committed, scored_runs, imports) ) if problems: return problems @@ -985,7 +1097,8 @@ def append_only_problems( new = current.get("entries") for entries in (old, new): if not isinstance(entries, list) or not all( - isinstance(entry, Mapping) for entry in entries + isinstance(entry, Mapping) and isinstance(entry.get("id"), str) + for entry in entries ): return [*problems, "registry: entries are malformed"] old_ids = [entry.get("id") for entry in old] @@ -1117,7 +1230,7 @@ def history_problems( *, full: bool = True, scored_runs: Mapping[str, tuple[str, str]] = SCORED_RUNS, - import_gates: frozenset[str] = HISTORICAL_IMPORT_GATES, + imports: Mapping[str, Mapping[str, Any]] = HISTORICAL_IMPORTS, ) -> list[str]: """Problems across every first-parent commit from ``base`` to ``head``, so a run of commits cannot hide a change a later commit @@ -1169,7 +1282,7 @@ def history_problems( vocabulary=sources if isinstance(sources, Mapping) else {}, commit=commit, scored_runs=scored_runs, - import_gates=import_gates, + imports=imports, ) if shape: problems.extend(f"{label}: {p}" for p in shape) diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py index ecad1e9e4..a5e073e21 100644 --- a/tests/test_track_b_holdout_registry.py +++ b/tests/test_track_b_holdout_registry.py @@ -177,12 +177,35 @@ def make_registry(entries: list[dict], complete=False) -> dict: } -#: The invented scored runs and historical-import gates, in place of the -#: module's controlled lists. Without a repository no pin is checked. +#: The invented scored runs, in place of the module's controlled list. +#: Without a repository no pin is checked. SCORED_RUNS = { run_path(f"s{index}"): (f"gate_{index}", "0" * 64) for index in range(12) } -IMPORT_GATES = frozenset(f"gate_{index}" for index in range(12)) + + +def authorize(current) -> dict: + """Authorize every well-formed historical import ``current`` holds as + it stands, for tests about other rules; the authorization rule itself + is tested against fixed authorizations.""" + entries = current.get("entries") if isinstance(current, dict) else None + imports = {} + for entry in entries if isinstance(entries, list) else []: + try: + if entry["provenance"] != "historical_import": + continue + imports[entry["id"]] = { + "gate": entry["gate"], + "unit_keys": tuple(entry["unit_keys"]), + "scored_runs": tuple( + item["path"] for item in entry["scored_evidence"] + ), + } + except (KeyError, TypeError): + continue + return imports + + #: The legal transitions, specified independently of the module. EXPECTED_TRANSITIONS = { "candidate": {"frozen", "withdrawn"}, @@ -192,14 +215,16 @@ def make_registry(entries: list[dict], complete=False) -> dict: } -def validate(current, repository=None, scored_runs=None, commit="HEAD"): +def validate( + current, repository=None, scored_runs=None, commit="HEAD", imports=None +): return registry.validate( current, repository, vocabulary=VOCABULARY, commit=commit, scored_runs=SCORED_RUNS if scored_runs is None else scored_runs, - import_gates=IMPORT_GATES, + imports=authorize(current) if imports is None else imports, ) @@ -625,12 +650,70 @@ def test_terminal_and_repeated_statuses_are_refused(statuses): ) -def test_only_listed_gates_may_be_imported(): +#: One fixed authorization: H001 imports gate_0's s_a:2020 with run s0. +AUTHORIZED = { + "H001": { + "gate": "gate_0", + "unit_keys": ("s_a:2020",), + "scored_runs": (run_path("s0"),), + } +} + + +def import_with(change) -> dict: + """gate_0's authorized import, then ``change`` applied to its entry.""" entry = make_entry(0, ["s_a:2020"], ("historical_import", "used")) - entry["gate"] = "gate_99" - entry["gate_identifiers"] = ["gate_99"] - problems = validate(make_registry([entry])) - assert any("scored before the registry existed" in p for p in problems) + change(entry) + return make_registry([entry]) + + +def _rename(entry): + entry["id"] = "H002" + + +def _regate(entry): + entry.update(gate="gate_1", gate_identifiers=["gate_1"]) + entry["scored_evidence"][0].update(path=run_path("s1"), equals="gate_1") + + +def _more_units(entry): + entry["unit_keys"].append("s_c:2099") + + +def _other_units(entry): + entry["unit_keys"] = ["s_c:2099"] + + +def _more_runs(entry): + entry["scored_evidence"].append(dict(entry["scored_evidence"][0])) + + +@pytest.mark.parametrize( + "change, message", + [ + (_rename, "not an authorized historical import"), + (_regate, "is authorized for 'gate_0'"), + (_more_units, "unit keys differ"), + (_other_units, "unit keys differ"), + (_more_runs, "differs from its authorized runs"), + ], +) +def test_imports_must_match_their_authorization(change, message): + assert validate(import_with(lambda entry: None), imports=AUTHORIZED) == [] + problems = validate(import_with(change), imports=AUTHORIZED) + assert any(message in p for p in problems), problems + + +def test_an_authorization_admits_one_import_only(): + first = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + again = make_entry(0, ["s_a:2020"], ("historical_import", "used")) + again["id"] = "H002" + entries = [first, again] + settle(entries) + problems = validate(make_registry(entries), imports=AUTHORIZED) + assert any( + "H002: is not an authorized historical import" in p for p in problems + ) def test_a_review_must_name_an_earlier_entry(): @@ -892,7 +975,75 @@ def test_a_registration_must_precede_its_scored_run( if registered_first: assert problems == [] else: - assert any("already committed when" in p for p in problems) + assert any("already committed" in p for p in problems) + + +def git_rename(repository: Path, old: str, new: str) -> None: + (repository / new).parent.mkdir(parents=True, exist_ok=True) + git(repository, "mv", old, new) + + +@pytest.mark.parametrize( + "scenario", ["moved", "deleted_and_restored", "same_commit", "side_branch"] +) +def test_a_score_committed_anywhere_before_registration_is_refused( + tmp_path, no_ci_env, scenario +): + """The score's bytes were committed before the registration in every + scenario: at another path then moved, at its path then deleted and + restored, in the registering commit itself, or on the base branch + while the registration waited on a side branch merged afterwards.""" + repository = init_repository(tmp_path) + listed, elsewhere = run_path("s0"), run_path("original") + if scenario == "moved": + write(repository, elsewhere, SCORE) + commit_all(repository, "score at another path") + elif scenario == "deleted_and_restored": + write(repository, listed, SCORE) + commit_all(repository, "score") + (repository / listed).unlink() + commit_all(repository, "score deleted") + if scenario == "same_commit": + write(repository, listed, SCORE) + commit_registry(repository, registered(["candidate"]), "both") + elif scenario == "side_branch": + git(repository, "checkout", "-q", "-b", "side") + commit_registry(repository, registered(["candidate"]), "register") + git(repository, "checkout", "-q", "master") + write(repository, listed, SCORE) + commit_all(repository, "score") + git(repository, "merge", "-q", "--no-ff", "-m", "merge side", "side") + else: + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + if scenario == "moved": + git_rename(repository, elsewhere, listed) + commit_all(repository, "moved to its listed path") + elif scenario == "deleted_and_restored": + write(repository, listed, SCORE) + commit_all(repository, "score restored") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + assert any("already committed" in p for p in problems), problems + + +def test_an_unanswerable_timing_check_fails_closed( + tmp_path, no_ci_env, monkeypatch +): + repository = init_repository(tmp_path) + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + assert validate(used, repository, pin(SCORE)) == [] + monkeypatch.setattr( + registry.Committed, "committed_by", lambda self, content, at: None + ) + problems = validate(used, repository, pin(SCORE)) + assert any("cannot tell whether" in p for p in problems) def test_a_used_registration_must_already_be_registered(tmp_path, no_ci_env): @@ -916,7 +1067,7 @@ def test_each_snapshot_is_validated_against_its_own_tree(tmp_path, no_ci_env): runs = pin(SCORE) assert validate(used, repository, runs) == [] problems = registry.history_problems( - repository, base, scored_runs=runs, import_gates=IMPORT_GATES + repository, base, scored_runs=runs, imports={} ) assert any("not a committed file" in p for p in problems) @@ -927,9 +1078,7 @@ def test_a_new_registry_is_admitted_against_an_empty_one(tmp_path, no_ci_env): commit_registry( repository, registered(["candidate", "frozen"]), "arrives frozen" ) - problems = registry.history_problems( - repository, base, import_gates=IMPORT_GATES - ) + problems = registry.history_problems(repository, base, imports={}) assert any("lone candidate" in p for p in problems) @@ -948,7 +1097,7 @@ def test_a_clean_history_has_no_problems(tmp_path, no_ci_env): repository, base, scored_runs=pin(SCORE), - import_gates=IMPORT_GATES, + imports={}, ) == [] ) @@ -970,9 +1119,7 @@ def test_a_run_of_commits_cannot_hide_a_change_it_undoes( commit_registry(repository, changed, "change") commit_registry(repository, first, "undo") assert registry.append_only_problems(first, first) == [] - assert registry.history_problems( - repository, base, full=full, import_gates=IMPORT_GATES - ) + assert registry.history_problems(repository, base, full=full, imports={}) def test_a_pull_request_head_is_read_from_its_event(tmp_path, monkeypatch): @@ -999,13 +1146,43 @@ def test_malformed_snapshots_are_reported_not_raised(tmp_path, no_ci_env): ) commit_all(repository, "a malformed snapshot") commit_registry(repository, registered(["candidate"]), "repaired") - problems = registry.history_problems( - repository, base, import_gates=IMPORT_GATES - ) + problems = registry.history_problems(repository, base, imports={}) assert len(problems) >= 3 assert registry.append_only_problems(registered(["candidate"]), None) +def test_malformed_predecessors_and_parser_limits_are_reported( + tmp_path, no_ci_env +): + no_id = {"entries": [{}]} + assert registry.append_only_problems( + no_id, {"entries": [{"units": {"years": [2020]}}]} + ) + repository = init_repository(tmp_path) + base = commit_registry(repository, no_id, "a malformed predecessor") + commit_registry(repository, registered(["candidate"]), "repaired") + for full in (True, False): + problems = registry.history_problems( + repository, base, full=full, imports={} + ) + assert isinstance(problems, list) + huge = "9" * 5000 + write(repository, registry.GATES, f"gates:\n gate_0:\n id: {huge}\n") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "an identifier past Python's integer limit") + assert isinstance(validate(imported(), repository, pin(SCORE)), list) + write(repository, registry.REGISTRY, '{"entries": [' + huge + "]}") + huge_json = commit_all(repository, "a registry past the integer limit") + with pytest.raises(ValueError, match="is not JSON"): + registry.registry_at(repository, huge_json) + nul = "HEAD\x00" + assert validate(imported(), repository, pin(SCORE), commit=nul) + with pytest.raises(LookupError): + registry.registry_at(repository, nul) + with pytest.raises(LookupError): + registry.history_problems(repository, nul, imports={}) + + def test_review_citations_must_name_committed_text_lines(tmp_path, no_ci_env): repository = init_repository(tmp_path) write(repository, "docs/note.md", "one\ntwo\nthree\n") diff --git a/tests/test_track_b_holdout_registry_committed.py b/tests/test_track_b_holdout_registry_committed.py index a2322ba58..a7e8b6e5f 100644 --- a/tests/test_track_b_holdout_registry_committed.py +++ b/tests/test_track_b_holdout_registry_committed.py @@ -79,12 +79,15 @@ "runs/gate_w1_candidate3_v1.json": "gate_w1", "runs/gate_w1_candidate4_v1.json": "gate_w1", } +#: The authorized historical imports: each id's gate. Its unit keys are +#: the seeded keys above, and its scored runs are all of its gate's listed +#: runs. EXPECTED_IMPORT_GATES = { - "gate_m6", - "B2", - "gate_epuf_fill", - "gate_m4", - "gate_w1", + "H001": "gate_m6", + "H002": "B2", + "H003": "gate_epuf_fill", + "H004": "gate_m4", + "H005": "gate_w1", } @@ -187,11 +190,30 @@ def test_the_vocabulary_and_registrations_are_as_specified(): } -def test_the_scored_runs_and_import_gates_are_as_specified(): +def test_the_scored_runs_and_imports_are_as_specified(): assert { path: gate for path, (gate, _) in registry.SCORED_RUNS.items() } == EXPECTED_SCORED_RUN_GATES - assert set(registry.HISTORICAL_IMPORT_GATES) == EXPECTED_IMPORT_GATES + expected = { + name: { + "gate": gate, + "unit_keys": sorted(EXPECTED_SEED_KEYS[name]), + "scored_runs": sorted( + path + for path, owner in EXPECTED_SCORED_RUN_GATES.items() + if owner == gate + ), + } + for name, gate in EXPECTED_IMPORT_GATES.items() + } + assert { + name: { + "gate": authorized["gate"], + "unit_keys": sorted(authorized["unit_keys"]), + "scored_runs": sorted(authorized["scored_runs"]), + } + for name, authorized in registry.HISTORICAL_IMPORTS.items() + } == expected for path, (_, digest) in registry.SCORED_RUNS.items(): content = (ROOT / path).read_bytes() assert hashlib.sha256(content).hexdigest() == digest, path diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 55de32e2a..94ebbf5d6 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 10048, + "unit": 10059, "artifact": 4263, "integration_psid": 1450, "reproduction_legacy": 520, From daf0242552805276bae912c95cfdffbaaaf5886c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 10:33:37 -0400 Subject: [PATCH 6/9] Holdout registry: answer review round 5 (one medium, three low) 1. Shallow history made the timing check fail open (medium). An empty find-object search is now a negative only in a complete repository: when git reports a shallow clone, committed_by returns None and the fail-closed diagnostic rejects the entry. A positive find is still conclusive. The search also runs with --no-replace-objects, so a replacement ref that orphans a commit cannot hide a committed score (the review's nonblocking environment-trust point). 2. A score added only in a merge resolution had no regression test (low). That topology is now tested; dropping -m from the search fails it. 3. Subprocess failures escaped (low). _git and _git_bytes turn TypeError and OSError (an argument list too long, a missing git, a None argument) into git-style failures, so validate reports a problem and registry_at and history_problems raise their documented LookupError. validate also checks the shape of injected import authorizations. 4. SHA-256 object ids were ignored at event boundaries (low). SHA_RX accepts 40- and 64-digit ids, so a SHA-256 push's before commit and a SHA-256 pull request head are honoured. Mutations: 70 of 70 killed, adding merge diffs not searched, shallow history trusted, replacement refs honoured, SHA-256 ids rejected, subprocess OSError escapes and authorization shapes unchecked. Tests: 130 passed, 2 skipped (with the birth-evidence test). Tier counts: unit 18,197 (+8 since round 4), artifact 4,263. Co-Authored-By: Claude Opus 5.5 --- .../track_b_holdout_registry_mutations.json | 90 +++++++++-- .../track_b/holdout_registry.py | 49 +++++- tests/README-tiers.md | 4 +- tests/test_track_b_holdout_registry.py | 149 ++++++++++++++++++ tests/tier_counts.json | 2 +- 5 files changed, 274 insertions(+), 20 deletions(-) diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json index 069638a4c..ea53d4a26 100644 --- a/docs/design/track_b_holdout_registry_mutations.json +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -4,7 +4,7 @@ "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", "mutation_method": "One deliberate production-line change per check; both registry test modules run in full and are expected to fail; original bytes restored before the next check. No mutated implementation is committed.", "restored_source_sha256": { - "src/populace_dynamics/track_b/holdout_registry.py": "0d82ef49bae6458c00e7c5386779ddad521b3eab60f63105600fb5c0f5bfec1b" + "src/populace_dynamics/track_b/holdout_registry.py": "e9879409df7b060bba1d1e379cb3afc844ba9a21e6339eecbc59f5e59dfea879" }, "checks": [ { @@ -64,6 +64,7 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_silent_shared_unit_is_refused", "tests/test_track_b_holdout_registry.py::test_keys_overlap_on_source_and_intersecting_years", "tests/test_track_b_holdout_registry.py::test_shared_units_match_the_independent_rule", "tests/test_track_b_holdout_registry_committed.py::test_the_seeded_disclosures_are_as_specified" @@ -248,10 +249,13 @@ "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]", + "tests/test_track_b_holdout_registry.py::test_a_replacement_ref_cannot_hide_a_committed_score", + "tests/test_track_b_holdout_registry.py::test_a_score_added_only_in_a_merge_resolution_is_refused", "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[deleted_and_restored]", "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[moved]", "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[same_commit]", - "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]" + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]", + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed" ] }, { @@ -261,21 +265,25 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed", "tests/test_track_b_holdout_registry.py::test_an_unanswerable_timing_check_fails_closed" ] }, { "mutation": "scores never found in history", - "old": "return bool(found.stdout.strip())", - "new": "return False", + "old": "if found.stdout.strip():\n return True", + "new": "if False:\n return True", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_registration_must_precede_its_scored_run[False]", + "tests/test_track_b_holdout_registry.py::test_a_replacement_ref_cannot_hide_a_committed_score", + "tests/test_track_b_holdout_registry.py::test_a_score_added_only_in_a_merge_resolution_is_refused", "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[deleted_and_restored]", "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[moved]", "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[same_commit]", - "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]" + "tests/test_track_b_holdout_registry.py::test_a_score_committed_anywhere_before_registration_is_refused[side_branch]", + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed" ] }, { @@ -320,8 +328,8 @@ }, { "mutation": "NUL arguments raise", - "old": "except ValueError as error:\n return subprocess.CompletedProcess(command, 128, \"\", str(error))", - "new": "except OSError as error:\n return subprocess.CompletedProcess(command, 128, \"\", str(error))", + "old": "except (ValueError, TypeError, OSError) as error:\n return subprocess.CompletedProcess(", + "new": "except (TypeError, OSError) as error:\n return subprocess.CompletedProcess(", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ @@ -609,7 +617,8 @@ "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-edit_then_restore]", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-edit_then_restore]", - "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused" + "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused", + "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" ] }, { @@ -650,7 +659,8 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ - "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit" + "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" ] }, { @@ -687,6 +697,68 @@ "tests/test_track_b_holdout_registry.py::test_each_snapshot_is_validated_against_its_own_tree" ] }, + { + "mutation": "merge diffs not searched", + "old": "\"-m\",\n \"-1\",", + "new": "\"-1\",", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_score_added_only_in_a_merge_resolution_is_refused" + ] + }, + { + "mutation": "shallow history trusted", + "old": "if shallow.returncode != 0 or shallow.stdout.strip() != \"false\":", + "new": "if False:", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed" + ] + }, + { + "mutation": "replacement refs honoured", + "old": "\"--no-replace-objects\",", + "new": "", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_replacement_ref_cannot_hide_a_committed_score" + ] + }, + { + "mutation": "SHA-256 ids rejected", + "old": "SHA_RX = re.compile(r\"[0-9a-f]{40}(?:[0-9a-f]{24})?\")", + "new": "SHA_RX = re.compile(r\"[0-9a-f]{40}\")", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" + ] + }, + { + "mutation": "subprocess OSError escapes", + "old": "except (ValueError, TypeError, OSError) as error:\n return subprocess.CompletedProcess(", + "new": "except ValueError as error:\n return subprocess.CompletedProcess(", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_subprocess_failures_are_reported_not_raised" + ] + }, + { + "mutation": "authorization shapes unchecked", + "old": "problems.extend(_authorization_problems(imports))", + "new": "pass", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_malformed_authorizations_are_reported_not_raised[None-must be a mapping]", + "tests/test_track_b_holdout_registry.py::test_malformed_authorizations_are_reported_not_raised[imports1-needs a gate]", + "tests/test_track_b_holdout_registry.py::test_malformed_authorizations_are_reported_not_raised[imports2-needs]" + ] + }, { "mutation": "a new registry needs no admission", "old": "before = previous if previous is not None else empty_like(current)", diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py index f864a71ca..7425454e6 100644 --- a/src/populace_dynamics/track_b/holdout_registry.py +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -358,7 +358,8 @@ r"(?:-(?P[0-9]{1,7}))?" ) URL_RX = re.compile(r"https://\S+") -SHA_RX = re.compile(r"[0-9a-f]{40}") +#: A full object id: SHA-1 (40 hex digits) or SHA-256 (64). +SHA_RX = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?") def load(repository: Path) -> dict[str, Any]: @@ -432,14 +433,15 @@ def lookup(document: Any, dotted: str) -> Any: def _git(repository: Path, *args: str) -> subprocess.CompletedProcess: - """Run git; an argument git cannot take (a NUL byte) fails like a git - error rather than raising.""" + """Run git. An argument git cannot take (a NUL byte, ``None``, an + argument list too long) or a missing git fails like a git error + rather than raising.""" command = ["git", "-C", str(repository), *args] try: return subprocess.run( command, capture_output=True, text=True, check=False ) - except ValueError as error: + except (ValueError, TypeError, OSError) as error: return subprocess.CompletedProcess(command, 128, "", str(error)) @@ -453,7 +455,7 @@ def _git_bytes( capture_output=True, check=False, ) - except ValueError: + except (ValueError, TypeError, OSError): return None return found.stdout if found.returncode == 0 else None @@ -578,7 +580,10 @@ def registered_at(self, entry_id: str) -> str | None: def committed_by(self, content: bytes, commit: str) -> bool | None: """Whether a file with exactly ``content`` was committed, at any path and whether or not it was later moved or deleted, in - ``commit`` or any of its ancestors. ``None`` when git cannot say.""" + ``commit`` or any of its ancestors. ``None`` when git cannot say, + including when a shallow clone hides ancestors: an empty search + proves nothing there. Replacement refs are ignored, so the search + reads the recorded history.""" name = _git_bytes( self.repository, "hash-object", "--stdin", stdin=content ) @@ -586,6 +591,7 @@ def committed_by(self, content: bytes, commit: str) -> bool | None: return None found = _git( self.repository, + "--no-replace-objects", "log", "-m", "-1", @@ -595,7 +601,12 @@ def committed_by(self, content: bytes, commit: str) -> bool | None: ) if found.returncode != 0: return None - return bool(found.stdout.strip()) + if found.stdout.strip(): + return True + shallow = _git(self.repository, "rev-parse", "--is-shallow-repository") + if shallow.returncode != 0 or shallow.stdout.strip() != "false": + return None + return False def _decode(content: bytes | None) -> str | None: @@ -607,6 +618,27 @@ def _decode(content: bytes | None) -> str | None: return None +def _authorization_problems(imports: Any) -> list[str]: + """Problems with the shape of the historical-import authorizations.""" + if not isinstance(imports, Mapping): + return ["imports: the authorizations must be a mapping"] + problems = [] + for name, authorized in imports.items(): + if not ( + isinstance(authorized, Mapping) + and isinstance(authorized.get("gate"), str) + and all( + isinstance(authorized.get(key), (list, tuple)) + and all(isinstance(item, str) for item in authorized[key]) + for key in ("unit_keys", "scored_runs") + ) + ): + problems.append( + f"imports: {name!r} needs a gate, unit_keys and scored_runs" + ) + return problems + + def _import_problems( entry: Mapping[str, Any], imports: Mapping[str, Mapping[str, Any]] ) -> list[str]: @@ -950,6 +982,7 @@ def validate( if not isinstance(registry["complete"], bool): problems.append("registry: complete must be true or false") problems.extend(_vocabulary_problems(vocabulary)) + problems.extend(_authorization_problems(imports)) if not isinstance(vocabulary, Mapping) or registry["sources"] != dict( vocabulary ): @@ -1190,7 +1223,7 @@ def registry_at(repository: Path, commit: str) -> Any: ``ValueError`` when the registry there is not JSON.""" listed = _git(repository, "ls-tree", "--name-only", commit, REGISTRY) if listed.returncode != 0: - raise LookupError(f"cannot list {REGISTRY} at {commit}") + raise LookupError(f"cannot list {REGISTRY} at {str(commit)[:80]}") if not listed.stdout.strip(): return None document = _parse_registry( diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 12e8fab41..0e605461d 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,9 +39,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 18,189 | +| `unit` | 18,197 | | `artifact` | 4,263 | | `integration_psid` | 1,450 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 308 | -| **Total** | **24,730** | +| **Total** | **24,738** | diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py index a5e073e21..0fb188b62 100644 --- a/tests/test_track_b_holdout_registry.py +++ b/tests/test_track_b_holdout_registry.py @@ -1028,6 +1028,155 @@ def test_a_score_committed_anywhere_before_registration_is_refused( assert any("already committed" in p for p in problems), problems +def test_a_shallow_clone_cannot_prove_a_score_was_not_committed( + tmp_path, no_ci_env +): + """A shallow clone that keeps the registry's history but not the + score's first commit must not read an empty search as absence.""" + repository = init_repository(tmp_path) + listed = run_path("s0") + write(repository, listed, SCORE) + commit_all(repository, "score") + (repository / listed).unlink() + commit_all(repository, "score deleted") + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, listed, SCORE) + commit_all(repository, "score restored") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + shallow = tmp_path / "shallow" + subprocess.run( + ["git", "clone", "-q", "--depth", "5", repository.as_uri(), shallow], + check=True, + capture_output=True, + ) + assert git(shallow, "rev-parse", "--is-shallow-repository").strip() == ( + "true" + ) + assert any( + "already committed" in p + for p in validate(used, repository, pin(SCORE)) + ) + problems = validate(used, shallow, pin(SCORE)) + assert any("cannot tell whether" in p for p in problems), problems + + +def test_a_replacement_ref_cannot_hide_a_committed_score(tmp_path, no_ci_env): + """`git replace` can graft a parentless copy over a commit and hide + its ancestors from ordinary log walks; the search ignores it.""" + repository = init_repository(tmp_path) + listed = run_path("s0") + write(repository, listed, SCORE) + commit_all(repository, "score") + (repository / listed).unlink() + deleted = commit_all(repository, "score deleted") + orphan = git( + repository, "commit-tree", f"{deleted}^{{tree}}", "-m", "orphan" + ).strip() + git(repository, "replace", deleted, orphan) + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, listed, SCORE) + commit_all(repository, "score restored") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + assert any("already committed" in p for p in problems), problems + + +def test_a_score_added_only_in_a_merge_resolution_is_refused( + tmp_path, no_ci_env +): + """A score that enters only through a merge commit's own resolution, + before the registration, was committed before it.""" + repository = init_repository(tmp_path) + git(repository, "checkout", "-q", "-b", "side") + write(repository, "docs/side.md", "side\n") + commit_all(repository, "side work") + git(repository, "checkout", "-q", "master") + write(repository, "docs/main.md", "main\n") + commit_all(repository, "main work") + git(repository, "merge", "-q", "--no-ff", "--no-commit", "side") + write(repository, run_path("s0"), SCORE) + git(repository, "add", "-A") + git(repository, "commit", "-q", "-m", "merge, adding the score") + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + problems = validate(used, repository, pin(SCORE)) + assert any("already committed" in p for p in problems), problems + + +def test_subprocess_failures_are_reported_not_raised( + tmp_path, no_ci_env, monkeypatch +): + repository = init_repository(tmp_path) + huge = "a" * 3_000_000 + current = registered(["candidate"]) + current.update(completion_record=huge, complete=True) + assert isinstance(validate(current, repository), list) + for commit in (None, huge): + with pytest.raises(LookupError): + registry.registry_at(repository, commit) + with pytest.raises(LookupError): + registry.history_problems(repository, huge, imports={}) + monkeypatch.setenv("PATH", str(tmp_path / "no-git-here")) + with pytest.raises(LookupError): + registry.registry_at(repository, "HEAD") + assert validate(imported(), repository, pin(SCORE)) + + +@pytest.mark.parametrize( + "imports, message", + [ + (None, "must be a mapping"), + ({"H001": {}}, "needs a gate"), + ( + {"H001": {"gate": "g", "unit_keys": [1], "scored_runs": []}}, + "needs", + ), + ], +) +def test_malformed_authorizations_are_reported_not_raised(imports, message): + problems = registry.validate( + imported(), vocabulary=VOCABULARY, imports=imports + ) + assert any(message in p for p in problems), problems + + +def test_sha256_object_ids_bound_histories(tmp_path, no_ci_env, monkeypatch): + repository = tmp_path / "repo256" + repository.mkdir() + git(repository, "init", "-q", "-b", "master", "--object-format=sha256") + git(repository, "config", "user.email", "t@example.com") + git(repository, "config", "user.name", "t") + first = registered(["candidate"]) + before = commit_registry(repository, first, "registry") + assert len(before) == 64 + changed = copy.deepcopy(first) + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "an immutable key changes") + commit_registry(repository, first, "and is restored") + commit_all(repository, "an unchanged tip") + event = tmp_path / "event.json" + event.write_text( + json.dumps( + {"before": before, "pull_request": {"head": {"sha": before}}} + ), + encoding="utf-8", + ) + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + assert registry.base_commit(repository) == before + assert registry.history_problems( + repository, registry.base_commit(repository), imports={} + ) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + assert registry.pull_request_head() == before + + def test_an_unanswerable_timing_check_fails_closed( tmp_path, no_ci_env, monkeypatch ): diff --git a/tests/tier_counts.json b/tests/tier_counts.json index c23ff83bb..4b3165560 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 18189, + "unit": 18197, "artifact": 4263, "integration_psid": 1450, "reproduction_legacy": 520, From 9ef16432d49afd8f912ebe006578ec46cc9ab90c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 07:41:11 -0400 Subject: [PATCH 7/9] Holdout registry: a push or pull request boundary never falls back (review round 6) Round 6 (Astra, at daf02425) found one medium and one low. - Medium: a transient git failure while resolving a push's `before` commit made `base_commit` fall back to the merge base, which on master is HEAD^1, so the history check covered fewer commits and passed an edit-and-restore of immutable unit keys. A push now resolves its own `before` commit or raises LookupError; an unreadable event, or one naming no commit id, also raises. An all-zero `before` (a branch's first push) still falls through to the merge base. - The same class on pull requests: `pull_request_head` returned None for an unreadable event, and the committed-registry test then skipped the pull request's own commits. On a pull_request event it now raises. One helper, `_event`, reads the payload for both. - Low: `_git_bytes`'s TypeError and OSError handling had no test. A new test makes `hash-object` fail four ways and requires `committed_by` to return None and validation to report the timing as unanswerable. Tests: 137 passed, 2 skipped (20 new) in the two registry modules. Co-Authored-By: Claude Opus 5.5 --- .../track_b/holdout_registry.py | 58 ++++++--- tests/test_track_b_holdout_registry.py | 122 ++++++++++++++++++ 2 files changed, 161 insertions(+), 19 deletions(-) diff --git a/src/populace_dynamics/track_b/holdout_registry.py b/src/populace_dynamics/track_b/holdout_registry.py index 7425454e6..fd2f7e8b1 100644 --- a/src/populace_dynamics/track_b/holdout_registry.py +++ b/src/populace_dynamics/track_b/holdout_registry.py @@ -1178,29 +1178,48 @@ def _commit(repository: Path, ref: str) -> str | None: return found.stdout.strip() if found.returncode == 0 else None +def _event() -> Mapping[str, Any]: + """The GitHub event payload at ``GITHUB_EVENT_PATH``. Raises + ``LookupError`` when it cannot be read or is not a JSON object.""" + path = os.environ.get("GITHUB_EVENT_PATH") + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except (TypeError, ValueError, OSError) as error: + raise LookupError(f"cannot read the GitHub event: {error}") from None + if not isinstance(payload, Mapping): + raise LookupError("the GitHub event is not a JSON object") + return payload + + def base_commit(repository: Path) -> str: """The commit the registry is compared against. - On a push (``GITHUB_EVENT_NAME=push``), the event's ``before`` - commit, which was the branch tip before the push. + commit, which was the branch tip before the push. An all-zero + ``before`` (a branch's first push) falls through to the merge base. - Otherwise, the merge base of ``HEAD`` and the base branch (``GITHUB_BASE_REF`` on a pull request, otherwise ``master``), preferring its ``origin/`` ref. - When that merge base is ``HEAD`` itself, as on the base branch, its first parent. - Raises ``LookupError`` when none of these resolves.""" + Raises ``LookupError`` when none of these resolves, and on a push + whose event cannot be read or whose ``before`` commit does not + resolve: a push is compared with its own boundary, never with a + nearer commit that would check less history.""" head = _commit(repository, "HEAD") if head is None: raise LookupError(f"no HEAD in {repository}") if os.environ.get("GITHUB_EVENT_NAME") == "push": - event = os.environ.get("GITHUB_EVENT_PATH") - before = None - if event and Path(event).is_file(): - before = json.loads(Path(event).read_text()).get("before") - if isinstance(before, str) and SHA_RX.fullmatch(before): - if set(before) != {"0"} and _commit(repository, before): - return before + before = _event().get("before") + if not isinstance(before, str) or SHA_RX.fullmatch(before) is None: + raise LookupError("the push event names no before commit") + if set(before) != {"0"}: + if _commit(repository, before) is None: + raise LookupError( + f"the push's before commit {before} does not resolve" + ) + return before branch = os.environ.get("GITHUB_BASE_REF") or "master" for ref in (f"origin/{branch}", branch): found = _git(repository, "merge-base", "HEAD", ref) @@ -1241,19 +1260,20 @@ def base_registry(repository: Path) -> Any: def pull_request_head() -> str | None: - """The pull request's head commit, on a ``pull_request`` event.""" + """The pull request's head commit, on a ``pull_request`` event, and + ``None`` on any other. Raises ``LookupError`` on a ``pull_request`` + event that cannot be read or names no head commit, so the pull + request's own commits are never silently left unchecked.""" if os.environ.get("GITHUB_EVENT_NAME") != "pull_request": return None - event = os.environ.get("GITHUB_EVENT_PATH") - if not event or not Path(event).is_file(): - return None - try: - payload = json.loads(Path(event).read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return None - head = (payload.get("pull_request") or {}).get("head") or {} + pull_request = _event().get("pull_request") + head = ( + pull_request.get("head") if isinstance(pull_request, Mapping) else None + ) sha = head.get("sha") if isinstance(head, Mapping) else None - return sha if isinstance(sha, str) and SHA_RX.fullmatch(sha) else None + if not isinstance(sha, str) or SHA_RX.fullmatch(sha) is None: + raise LookupError("the pull_request event names no head commit") + return sha def history_problems( diff --git a/tests/test_track_b_holdout_registry.py b/tests/test_track_b_holdout_registry.py index 0fb188b62..2deea18cb 100644 --- a/tests/test_track_b_holdout_registry.py +++ b/tests/test_track_b_holdout_registry.py @@ -11,8 +11,10 @@ from __future__ import annotations import copy +import errno import hashlib import json +import os import subprocess from pathlib import Path @@ -1195,6 +1197,40 @@ def test_an_unanswerable_timing_check_fails_closed( assert any("cannot tell whether" in p for p in problems) +@pytest.mark.parametrize( + "failure", + [ + OSError(errno.EMFILE, "Too many open files"), + FileNotFoundError(errno.ENOENT, "git"), + TypeError("expected str, bytes or os.PathLike object"), + ValueError("embedded null byte"), + ], +) +def test_a_failed_object_hash_fails_closed( + tmp_path, no_ci_env, monkeypatch, failure +): + repository = init_repository(tmp_path) + commit_registry(repository, registered(["candidate"]), "register") + commit_registry(repository, registered(["candidate", "frozen"]), "freeze") + write(repository, run_path("s0"), SCORE) + commit_all(repository, "score") + used = registered(["candidate", "frozen", "used"]) + commit_registry(repository, used, "used") + assert validate(used, repository, pin(SCORE)) == [] + run = subprocess.run + + def failing(command, *args, **kwargs): + if "hash-object" in command: + raise failure + return run(command, *args, **kwargs) + + monkeypatch.setattr(registry.subprocess, "run", failing) + committed = registry.Committed(repository) + assert committed.committed_by(json.dumps(SCORE).encode(), "HEAD") is None + problems = validate(used, repository, pin(SCORE)) + assert any("cannot tell whether" in p for p in problems), problems + + def test_a_used_registration_must_already_be_registered(tmp_path, no_ci_env): repository = init_repository(tmp_path) write(repository, run_path("s0"), SCORE) @@ -1284,6 +1320,32 @@ def test_a_pull_request_head_is_read_from_its_event(tmp_path, monkeypatch): assert registry.pull_request_head() is None +@pytest.mark.parametrize( + "payload", + [ + None, + "not json", + "[]", + json.dumps({}), + json.dumps({"pull_request": None}), + json.dumps({"pull_request": {"head": "abc"}}), + json.dumps({"pull_request": {"head": {"sha": "HEAD"}}}), + ], +) +def test_an_unreadable_pull_request_event_raises( + tmp_path, monkeypatch, payload +): + event = tmp_path / "event.json" + if payload is not None: + event.write_text(payload, encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + with pytest.raises(LookupError): + registry.pull_request_head() + monkeypatch.setenv("GITHUB_EVENT_NAME", "workflow_dispatch") + assert registry.pull_request_head() is None + + def test_malformed_snapshots_are_reported_not_raised(tmp_path, no_ci_env): repository = init_repository(tmp_path) base = commit_registry(repository, registered(["candidate"]), "registry") @@ -1418,6 +1480,66 @@ def test_a_push_compares_with_its_before_commit( assert registry.base_registry(repository) == middle +@pytest.mark.parametrize( + "payload", + [ + None, + "not json", + "[]", + json.dumps({}), + json.dumps({"before": None}), + json.dumps({"before": "HEAD"}), + json.dumps({"before": "b" * 40}), + ], +) +def test_a_push_without_a_resolvable_before_raises( + tmp_path, no_ci_env, monkeypatch, payload +): + repository = init_repository(tmp_path) + commit_registry(repository, registered(["candidate"]), "registry") + event = tmp_path / "event.json" + if payload is not None: + event.write_text(payload, encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + with pytest.raises(LookupError): + registry.base_commit(repository) + with pytest.raises(LookupError): + registry.base_registry(repository) + + +@pytest.mark.parametrize("errno_code", [errno.EAGAIN, errno.EMFILE]) +def test_a_transient_failure_at_the_push_boundary_raises( + tmp_path, no_ci_env, monkeypatch, errno_code +): + repository = init_repository(tmp_path) + first = registered(["candidate"]) + before = commit_registry(repository, first, "registry") + changed = copy.deepcopy(first) + changed["entries"][0]["unit_keys"] = ["s_c:2199"] + commit_registry(repository, changed, "an immutable key changes") + commit_registry(repository, first, "and is restored") + commit_all(repository, "an unchanged tip") + event = tmp_path / "event.json" + event.write_text(json.dumps({"before": before}), encoding="utf-8") + monkeypatch.setenv("GITHUB_EVENT_NAME", "push") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + assert registry.base_commit(repository) == before + assert registry.history_problems( + repository, registry.base_commit(repository), imports={} + ) + run = subprocess.run + + def flaky(command, *args, **kwargs): + if f"{before}^{{commit}}" in command: + raise OSError(errno_code, os.strerror(errno_code)) + return run(command, *args, **kwargs) + + monkeypatch.setattr(registry.subprocess, "run", flaky) + with pytest.raises(LookupError): + registry.base_commit(repository) + + def test_no_base_raises_rather_than_passing(tmp_path, monkeypatch): monkeypatch.delenv("GITHUB_EVENT_NAME", raising=False) monkeypatch.setenv("GITHUB_BASE_REF", "no-such-branch") From fada81e2e33d8e3064e5885ea6ee4c643bb9ea9c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 10:50:36 -0400 Subject: [PATCH 8/9] Holdout registry: recount tiers after merging master (unit 18,217) Round 6 added 20 unit tests; master (2195258c) adds none. Co-Authored-By: Claude Opus 5.5 --- tests/README-tiers.md | 4 ++-- tests/tier_counts.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/README-tiers.md b/tests/README-tiers.md index 0e605461d..6f7b03ec2 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -39,9 +39,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 18,197 | +| `unit` | 18,217 | | `artifact` | 4,263 | | `integration_psid` | 1,450 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 308 | -| **Total** | **24,738** | +| **Total** | **24,758** | diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 4b3165560..56cfea1c2 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 18197, + "unit": 18217, "artifact": 4263, "integration_psid": 1450, "reproduction_legacy": 520, From 404a7ee66fd0222aec9e29117632b97507690864 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 11:23:24 -0400 Subject: [PATCH 9/9] Holdout registry: rerun the mutation record on the merged tree All 75 checks again fail the registry tests with the mutation and pass without it, against the merged source (a1fd1d99). The runner now records parametrized test ids in full; the previous record cut them at the first space, so some failing cases were missing from its lists. Co-Authored-By: Claude Opus 5.5 --- .../track_b_holdout_registry_mutations.json | 76 ++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/design/track_b_holdout_registry_mutations.json b/docs/design/track_b_holdout_registry_mutations.json index ea53d4a26..7e011ba97 100644 --- a/docs/design/track_b_holdout_registry_mutations.json +++ b/docs/design/track_b_holdout_registry_mutations.json @@ -4,7 +4,7 @@ "admitted_scope": "The held-out data registry's rules on invented registries, temporary git repositories and the committed registry; no population, behavioral or forecast claim.", "mutation_method": "One deliberate production-line change per check; both registry test modules run in full and are expected to fail; original bytes restored before the next check. No mutated implementation is committed.", "restored_source_sha256": { - "src/populace_dynamics/track_b/holdout_registry.py": "e9879409df7b060bba1d1e379cb3afc844ba9a21e6339eecbc59f5e59dfea879" + "src/populace_dynamics/track_b/holdout_registry.py": "a1fd1d999f59abecb6a8ca08801a7b2bbf7c4960cf3581f8c77e6cf2972ee354" }, "checks": [ { @@ -265,6 +265,10 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure0]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure1]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure2]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure3]", "tests/test_track_b_holdout_registry.py::test_a_shallow_clone_cannot_prove_a_score_was_not_committed", "tests/test_track_b_holdout_registry.py::test_an_unanswerable_timing_check_fails_closed" ] @@ -604,6 +608,7 @@ "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]", + "tests/test_track_b_holdout_registry.py::test_any_json_in_any_field_is_reported_not_raised", "tests/test_track_b_holdout_registry.py::test_removing_or_reordering_entries_is_refused", "tests/test_track_b_holdout_registry.py::test_the_base_branch_compares_with_its_parent" ] @@ -617,6 +622,8 @@ "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-edit_then_restore]", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-edit_then_restore]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]", "tests/test_track_b_holdout_registry.py::test_changing_an_earlier_entry_is_refused", "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" ] @@ -654,12 +661,15 @@ }, { "mutation": "push ignores its before commit", - "old": "if set(before) != {\"0\"} and _commit(repository, before):", + "old": "if set(before) != {\"0\"}:", "new": "if False:", "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ "tests/test_track_b_holdout_registry.py::test_a_push_compares_with_its_before_commit", + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"}]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]", "tests/test_track_b_holdout_registry.py::test_sha256_object_ids_bound_histories" ] }, @@ -744,6 +754,8 @@ "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", "exit_code": 1, "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]", "tests/test_track_b_holdout_registry.py::test_subprocess_failures_are_reported_not_raised" ] }, @@ -780,6 +792,66 @@ "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[False-append_then_remove]", "tests/test_track_b_holdout_registry.py::test_a_run_of_commits_cannot_hide_a_change_it_undoes[True-append_then_remove]" ] + }, + { + "mutation": "an unresolved push boundary falls back", + "old": " if _commit(repository, before) is None:\n raise LookupError(\n f\"the push's before commit {before} does not resolve\"\n )\n return before", + "new": " if _commit(repository, before) is not None:\n return before", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"}]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[24]", + "tests/test_track_b_holdout_registry.py::test_a_transient_failure_at_the_push_boundary_raises[35]" + ] + }, + { + "mutation": "a push naming no commit falls through", + "old": " raise LookupError(\"the push event names no before commit\")", + "new": " before = \"0\" * 40", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": \"HEAD\"}]", + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{\"before\": null}]", + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[{}]" + ] + }, + { + "mutation": "a non-object event is read", + "old": " if not isinstance(payload, Mapping):\n raise LookupError(\"the GitHub event is not a JSON object\")\n", + "new": "", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_push_without_a_resolvable_before_raises[[]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[[]" + ] + }, + { + "mutation": "an unreadable pull request event is skipped", + "old": " raise LookupError(\"the pull_request event names no head commit\")", + "new": " return None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{\"pull_request\": null}]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{\"pull_request\": {\"head\": \"abc\"}}]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{\"pull_request\": {\"head\": {\"sha\": \"HEAD\"}}}]", + "tests/test_track_b_holdout_registry.py::test_an_unreadable_pull_request_event_raises[{}]" + ] + }, + { + "mutation": "binary git failures escape", + "old": " except (ValueError, TypeError, OSError):\n return None\n return found.stdout if found.returncode == 0 else None", + "new": " except ValueError:\n return None\n return found.stdout if found.returncode == 0 else None", + "command": "PYTHONPATH=$PWD/src OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_WAIT_POLICY=PASSIVE python -m pytest -q -p no:cacheprovider -rf tests/test_track_b_holdout_registry.py tests/test_track_b_holdout_registry_committed.py", + "exit_code": 1, + "failed_tests": [ + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure0]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure1]", + "tests/test_track_b_holdout_registry.py::test_a_failed_object_hash_fails_closed[failure2]" + ] } ] }