From 2e77b1fa8d06d6ad155c0a9c25e329b2e3dba03a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 07:44:18 -0400 Subject: [PATCH 01/12] Add the compatibility_recertification release type (WIP) Data module only: parent identity from the parent's published manifest, the reviewed register of engine column changes (both directions), the loader replay under the tested wheels, drift metric definitions and combination, the validator, certification and the publish-time parent check. Builder tool, contract/release wiring, tests and docs follow. Co-Authored-By: Claude Opus 5.5 --- .../data/compatibility_recertification.py | 2173 +++++++++++++++++ 1 file changed, 2173 insertions(+) create mode 100644 packages/microcosm-data/src/microcosm/data/compatibility_recertification.py diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py new file mode 100644 index 000000000..db847a917 --- /dev/null +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -0,0 +1,2173 @@ +"""Re-certify a published release for a newer country model, bytes unchanged. + +A ``compatibility_recertification`` release is a child of one published, +certified US calibration release. It republishes the parent's microdata and +every other parent artifact byte for byte under a new id, and adds what the +parent cannot say for itself: that the same bytes were loaded and scored under +a newer policyengine-us, what that engine reads and ignores differently, and +how far the headline results moved. It transforms no data and never +recalibrates. + +**Parent identity comes from the parent's own published manifest.** The child +carries the parent's ``release_manifest.json`` and ``build_manifest.json`` +byte for byte (``parent_release_manifest.json``, ``parent_build_manifest.json``) +and records their SHA-256 in ``compatibility_recertification.json``. That +manifest pins every parent artifact's digest, so it is the whole lineage; no +parent is a code constant. Validation is offline: it checks the child against +the carried manifest and replays the current release contract on the parent +reconstructed from the carried files. Publication adds the online anchor +(:func:`require_parent_publication`): before any upload, the parent tag on the +Hub must serve exactly the carried manifest bytes. + +**What is measured, and how it is bound.** + +- *Loader replay* (:func:`run_recertification_loader_replay`). Under the + tested country, Core, wrapper and calculator wheels, both native loaders read + the H5. Every stored column is classified against the tested engine and the + stored-input register (:mod:`microcosm.data.stored_inputs`). A column the + built-with engine read and the tested engine ignores, or one the register + says no engine reads and the tested engine now reads, must have an entry in + :data:`RECERTIFICATION_COLUMN_CHANGES`, the reviewed register of such + changes, or the replay refuses. The receipt is replayed at every later gate. +- *Drift* (:func:`combine_drift`). The publisher's tool scores the H5 once + under the built-with engine and once under the tested engine (and once more + with the newly read inputs withheld), with the same household-batched scorer + and the same metric definitions (:func:`drift_metric_definitions`): the 18 + exact US critical rows, every scorable national SOI Table 1.4 dollar row and + the headline aggregates in :data:`HEADLINE_METRICS`. The drift document + embeds all score documents, so every row recomputes from them; the tested + side's runtime must be the exact runtime the loader replay certified. +- *Claim*. ``compatible_model_packages`` keeps every parent entry and gains one + ``publisher_claim`` entry for the tested version, attributed to a named + declarer. ``compatibility.claim_measurement`` records the one version + measured, the baseline it was compared with and the drift summary. Whether + the drift is acceptable is the declarer's call, not this module's. + +**Scope.** National-default parents only: :data:`SUPPORTED_PARENT_ROLES`. The +identity mechanism does not depend on the role, so a local-area parent is an +additive follow-up. A parent must itself be a calibration release; a +recertification of a recertification or of a source enrichment is refused, so +every child is one hop from the build its calibration evidence describes. + +See ``docs/us-compatibility-recertification.md``. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +import shutil +import tempfile +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from types import MappingProxyType + +from microcosm.data.contract import ( + PUBLISHER_CLAIM_BASIS, + ReleaseContractError, + required_release_files, +) + +RECERTIFICATION_RELEASE_TYPE = "compatibility_recertification" +REPORT_FILE = "compatibility_recertification.json" +RECEIPT_FILE = "compatibility_recertification_loader.json" +DRIFT_FILE = "compatibility_drift.json" +PARENT_RELEASE_MANIFEST_FILE = "parent_release_manifest.json" +PARENT_BUILD_MANIFEST_FILE = "parent_build_manifest.json" +REPORT_SCHEMA_VERSION = 1 +RECEIPT_SCHEMA_VERSION = 1 +SCORES_SCHEMA_VERSION = 1 +DRIFT_SCHEMA_VERSION = 1 +SCORES_KIND = "compatibility_recertification_scores" +DRIFT_KIND = "compatibility_drift" +REPO_ID = "policyengine/populace-us" +US_RELEASE_PREFIX = "populace-us-" +NATIONAL_DEFAULT_ROLE = "national_default" +#: Roles a parent may declare. The identity checks do not depend on the role; +#: the local-area contract's own replay is what a second role needs. +SUPPORTED_PARENT_ROLES = (NATIONAL_DEFAULT_ROLE,) +#: The parent's ``release_type``: absent or ``calibration``, never a child. +PARENT_RELEASE_TYPES = (None, "calibration") +#: Files the child writes itself; a parent release file by one of these names +#: is refused rather than shadowed. +CHILD_OWNED_FILES = ( + "release_manifest.json", + "build_manifest.json", + REPORT_FILE, + RECEIPT_FILE, + DRIFT_FILE, + PARENT_RELEASE_MANIFEST_FILE, + PARENT_BUILD_MANIFEST_FILE, +) +#: The child's own artifacts, by manifest key, besides the carried ones. +CHILD_ARTIFACT_KEYS = { + "build_manifest": "build_manifest.json", + "compatibility_recertification": REPORT_FILE, + "parent_release_manifest": PARENT_RELEASE_MANIFEST_FILE, + "parent_build_manifest": PARENT_BUILD_MANIFEST_FILE, +} +CERTIFIED_ARTIFACT_KEYS = { + "compatibility_recertification_loader": RECEIPT_FILE, + "compatibility_drift": DRIFT_FILE, +} +COMPATIBILITY_PACKAGES = ( + "policyengine-us", + "policyengine-core", + "policyengine", + "spm-calculator", +) +#: The producer inventory the release assembler, scorer and certifier run +#: from. Publication authenticates every file against the recorded commit and +#: the checkout (``source_enrichment._check_producer_source_identity``). +PRODUCER_SOURCE_FILES = ( + "tools/build_us_compatibility_recertification.py", + "packages/microcosm-data/src/microcosm/data/compatibility_recertification.py", + "packages/microcosm-data/src/microcosm/data/source_enrichment.py", + "packages/microcosm-data/src/microcosm/data/stored_inputs.py", + "packages/microcosm-data/src/microcosm/data/contract.py", +) +SCORING_PERIOD = 2024 +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_ENTITIES = ("person", "household", "tax_unit", "spm_unit", "family", "marital_unit") + + +# --------------------------------------------------------------------------- +# The reviewed register of what a newer engine reads and ignores differently. +# --------------------------------------------------------------------------- + +#: The column the built-with engine read is no longer a variable. +CHANGE_IGNORED = "ignored" +#: The register says no engine reads the column; the tested engine now does. +CHANGE_READ = "read" + + +@dataclass(frozen=True) +class ReviewedColumnChange: + """One reviewed change in how an engine treats a stored column.""" + + change: str + reason: str + + +_REMOVED_MORTGAGE_INPUT = ( + "A tax-unit float input of policyengine-us 2.2.1, defined in " + "variables/household/expense/tax_unit/mortgage_interest_structure.py. " + "PolicyEngine/policyengine-us#9605 (merged 2026-10-12) removed it; " + "policyengine-us 2.60.4 defines no variable of this name, and its own " + "tests/microsimulation/data/test_removed_mortgage_interest_inputs.py " + "states that a dataset carrying it loads with core's unknown-column " + "warning and computes mortgage interest from the person-level " + "home_mortgage_interest input alone. Releases store that input too, so " + "the interest is not lost; the {which} split of it is." +) + +#: Reviewed changes, keyed by stored column. A recertification refuses any +#: change the tested engine makes that has no entry here, and any entry whose +#: direction disagrees with what the engine does. An entry is evidence about +#: the engines named in its reason; adding one is a reviewed change. +RECERTIFICATION_COLUMN_CHANGES: Mapping[str, ReviewedColumnChange] = MappingProxyType( + { + "first_home_mortgage_interest": ReviewedColumnChange( + CHANGE_IGNORED, _REMOVED_MORTGAGE_INPUT.format(which="first-lien") + ), + "second_home_mortgage_interest": ReviewedColumnChange( + CHANGE_IGNORED, _REMOVED_MORTGAGE_INPUT.format(which="second-lien") + ), + "tax_unit_role_input": ReviewedColumnChange( + CHANGE_READ, + "The HEAD / SPOUSE / DEPENDENT role Microcosm's tax-unit " + "construction assigns each person (the stored-input register's " + "entry). policyengine-us 2.2.1 defines no variable of this name and " + "infers heads and spouses by age. PolicyEngine/policyengine-us#9617 " + '("Honor dataset-supplied tax-unit roles") made it a person Enum ' + "input: in 2.60.4, tax_unit_roles_supplied is true when every member " + "of a tax unit has a role, and is_tax_unit_head and " + "is_tax_unit_spouse then follow the stored role instead of age. It " + "raises on a unit with some members' roles missing, or without " + "exactly one HEAD and at most one SPOUSE.", + ), + **{ + column: ReviewedColumnChange( + CHANGE_READ, + f"The person_id of the co-resident parent the CPS ASEC {pointer} " + "pointer names, or 0 (the stored-input register's entry). " + "policyengine-us 2.2.1 defines no variable of this name. " + "PolicyEngine/policyengine-us#9406 (merged 2026-10-07) made it a " + "person int input; in 2.60.4 Medicaid's MAGI household " + "composition (variables/gov/hhs/medicaid/income/" + "_non_filer_household.py and _claiming_tax_unit.py) and the " + "parent-link helpers read it.", + ) + for column, pointer in ( + ("parent_1_id", "PEPAR1"), + ("parent_2_id", "PEPAR2"), + ) + }, + } +) + + +def column_change_register_sha256( + register: Mapping[str, ReviewedColumnChange] = RECERTIFICATION_COLUMN_CHANGES, +) -> str: + """SHA-256 of the register as canonical JSON, recorded in each receipt.""" + + payload = json.dumps( + {column: asdict(entry) for column, entry in register.items()}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ) + return hashlib.sha256(payload.encode("ascii")).hexdigest() + + +def stored_column_changes( + stored_tables: Mapping[str, Iterable[str]], + *, + engine_variables: Iterable[str], + stored_input_register: Mapping[str, str], +) -> dict[str, list[dict]]: + """How the tested engine's reading of the stored columns differs. + + Pure: the stored tables, the tested engine's variable names and the + stored-input register in, two sorted lists out. + + - ``ignored``: model-named columns that are neither tested-engine variables + nor register entries. The stored-input rule refuses exactly these; a + parent that passed that rule under its built-with engine stored them as + inputs that engine read. + - ``read``: register entries (columns the register says the engine + ignores) that the tested engine defines, so it now reads them. + + Each item names the column and every table that stores it. + """ + + from microcosm.data.stored_inputs import is_model_named + + variables = frozenset(engine_variables) + tables_by_column: dict[str, set[str]] = {} + for table, columns in stored_tables.items(): + for column in columns: + tables_by_column.setdefault(column, set()).add(str(table)) + ignored, read = [], [] + for column in sorted(tables_by_column): + if not is_model_named(column): + continue + item = {"column": column, "tables": sorted(tables_by_column[column])} + if column not in variables and column not in stored_input_register: + ignored.append(item) + elif column in variables and column in stored_input_register: + read.append(item) + return {CHANGE_IGNORED: ignored, CHANGE_READ: read} + + +def reviewed_column_change_failures( + changes: Mapping[str, Sequence[Mapping]], + register: Mapping[str, ReviewedColumnChange] = RECERTIFICATION_COLUMN_CHANGES, +) -> list[str]: + """One failure per change without a matching reviewed entry; [] if all are.""" + + failures = [] + for direction in (CHANGE_IGNORED, CHANGE_READ): + for item in changes.get(direction, ()): + column = item["column"] + entry = register.get(column) + if entry is None: + failures.append( + f"stored column {column!r} ({', '.join(item['tables'])}) is " + f"{'ignored' if direction == CHANGE_IGNORED else 'read'} by " + "the tested engine differently from the stored-input " + "register, and RECERTIFICATION_COLUMN_CHANGES has no " + "reviewed entry for it. Review what the engine change does " + "to the column and add the entry with its reason." + ) + elif entry.change != direction: + failures.append( + f"stored column {column!r}: the reviewed register says the " + f"engine change makes it {entry.change!r}, but the tested " + f"engine has it {direction!r}." + ) + return failures + + +def recorded_column_changes( + changes: Mapping[str, Sequence[Mapping]], + register: Mapping[str, ReviewedColumnChange] = RECERTIFICATION_COLUMN_CHANGES, +) -> dict[str, list[dict]]: + """The changes with each reviewed reason attached, as the report records them.""" + + return { + direction: [ + {**dict(item), "reason": register[item["column"]].reason} + for item in changes.get(direction, ()) + ] + for direction in (CHANGE_IGNORED, CHANGE_READ) + } + + +# --------------------------------------------------------------------------- +# Small helpers. +# --------------------------------------------------------------------------- + + +def sha256_file(path: Path | str) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_sha256(value: object) -> str: + payload = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ) + return hashlib.sha256(payload.encode("ascii")).hexdigest() + + +def _mapping(value: object) -> Mapping: + return value if isinstance(value, Mapping) else {} + + +def _json(path: Path, failures: list[str]) -> dict: + try: + payload = json.loads(path.read_text()) + if not isinstance(payload, dict): + raise ValueError("expected an object") + return payload + except (OSError, ValueError) as exc: + failures.append(f"{path.name}: {exc}") + return {} + + +def write_json(path: Path, value: object) -> None: + """Write a release JSON document the way every lane does: sorted, indented.""" + + path.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + + +def is_recertification(release_dir: Path | str) -> bool: + """Whether ``release_dir``'s manifest declares this release type. + + A reporting probe for the publisher; it validates nothing, and an + unreadable manifest reads as ``False``. + """ + + try: + manifest = json.loads((Path(release_dir) / "release_manifest.json").read_text()) + except (OSError, ValueError): + return False + return ( + isinstance(manifest, Mapping) + and manifest.get("release_type") == RECERTIFICATION_RELEASE_TYPE + ) + + +# --------------------------------------------------------------------------- +# The parent, from its published manifest. +# --------------------------------------------------------------------------- + + +def describe_parent( + parent_release_dir: Path | str, parent_artifact_root: Path | str +) -> dict: + """The parent block a child records, read from the parent's published files. + + ``parent_release_dir`` holds the parent's ``releases//`` files as the + Hub serves them at the parent tag; ``parent_artifact_root`` holds its root + artifacts (the H5 and the calibration NPZ). Every artifact the manifest + declares must be present with the digest it pins, in exactly one of the + two places: a bare filename in the release directory is a release file, + anything else a root artifact. + + Raises: + ValueError: The parent is not a supported published release, or a + file is missing or differs from its pin. + """ + + parent_release_dir = Path(parent_release_dir) + parent_artifact_root = Path(parent_artifact_root) + manifest_path = parent_release_dir / "release_manifest.json" + build_path = parent_release_dir / "build_manifest.json" + manifest = json.loads(manifest_path.read_text()) + if not isinstance(manifest, Mapping): + raise ValueError("parent release_manifest.json must be an object") + parent_id = parent_release_dir.name + build = _mapping(manifest.get("build")) + if build.get("build_id") != parent_id: + raise ValueError( + "parent release directory must be named for the manifest's build.build_id" + ) + if not parent_id.startswith(US_RELEASE_PREFIX): + raise ValueError("only a US release can be recertified by this lane") + if manifest.get("release_type") not in PARENT_RELEASE_TYPES: + raise ValueError( + f"parent release_type {manifest.get('release_type')!r} is not a " + "calibration release; recertify the calibration release itself" + ) + role = manifest.get("dataset_role", NATIONAL_DEFAULT_ROLE) + if role not in SUPPORTED_PARENT_ROLES: + raise ValueError( + f"parent dataset_role {role!r} is not supported yet; this lane " + f"recertifies {SUPPORTED_PARENT_ROLES} releases" + ) + release_files, root_artifacts = {}, {} + for key, raw in _mapping(manifest.get("artifacts")).items(): + entry = _mapping(raw) + path, sha = entry.get("path"), entry.get("sha256") + if not isinstance(path, str) or not isinstance(sha, str): + raise ValueError(f"parent artifact {key!r} has no path and sha256") + if entry.get("revision") != parent_id or entry.get("repo_id") != REPO_ID: + raise ValueError( + f"parent artifact {key!r} must pin {REPO_ID} at the parent tag" + ) + if Path(path).name == path and (parent_release_dir / path).is_file(): + if path in CHILD_OWNED_FILES: + raise ValueError( + f"parent artifact {path!r} collides with a file the child writes" + ) + local, place = parent_release_dir / path, release_files + else: + local, place = parent_artifact_root / path, root_artifacts + if path in release_files or path in root_artifacts: + raise ValueError(f"parent declares artifact path {path!r} twice") + if not local.is_file() or sha256_file(local) != sha: + raise ValueError( + f"parent artifact {path!r} is missing or differs from its pin" + ) + place[path] = sha + national = _mapping(manifest.get("default_datasets")).get("national") + dataset = _mapping(_mapping(manifest.get("artifacts")).get(national)) + if dataset.get("kind") != "microdata" or dataset.get("path") not in root_artifacts: + raise ValueError( + "parent default_datasets.national must name a root microdata artifact" + ) + for name in required_release_files(parent_id): + if name in {"release_manifest.json", "build_manifest.json"}: + continue + if name not in release_files: + raise ValueError(f"parent manifest must declare its required file {name!r}") + return { + "build_id": parent_id, + "repo_id": REPO_ID, + "revision": parent_id, + "release_type": manifest.get("release_type", "calibration"), + "dataset_role": role, + "release_manifest_sha256": sha256_file(manifest_path), + "build_manifest_sha256": sha256_file(build_path), + "built_with_model_package": dict( + _mapping(build.get("built_with_model_package")) + ), + "built_with_core_package": dict(_mapping(build.get("built_with_core_package"))), + "release_files": dict(sorted(release_files.items())), + "root_artifacts": dict(sorted(root_artifacts.items())), + "dataset": { + "artifact": national, + "filename": dataset["path"], + "sha256": dataset["sha256"], + }, + } + + +def _reconstructed_parent(release_dir: Path, parent: Mapping, staging: Path) -> Path: + """The parent's release directory, rebuilt from the bytes the child carries.""" + + directory = staging / str(parent["build_id"]) + directory.mkdir() + shutil.copyfile( + release_dir / PARENT_RELEASE_MANIFEST_FILE, directory / "release_manifest.json" + ) + shutil.copyfile( + release_dir / PARENT_BUILD_MANIFEST_FILE, directory / "build_manifest.json" + ) + for name in _mapping(parent.get("release_files")): + shutil.copyfile(release_dir / name, directory / name) + return directory + + +#: The parent-contract outcome when every failure is the missing microcosm#1034 +#: stored-input evidence, which the recertification re-measures itself under +#: both engines (the baseline scores' and the loader replay's verdicts). +PARENT_CONTRACT_PASSED = "passed" +PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED = ( + "passed_with_stored_input_evidence_replaced" +) + + +def replay_parent_contract( + release_dir: Path | str, parent: Mapping, *, artifact_root: Path | str | None +) -> dict: + """Run today's release contract on the parent rebuilt from the carried bytes. + + A parent published before microcosm#1034 carries no ``stored_inputs.json`` + and fails only that check. The recertification measures the same rule + itself, against the current register, under the built-with engine (the + baseline scores) and the tested engine (the loader replay), so such a + parent passes as :data:`PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED`, + with the replaced failure lines recorded. Any other failure is returned in + ``other_failures`` and refuses the child. + """ + + from microcosm.data import contract + + release_dir = Path(release_dir) + with tempfile.TemporaryDirectory(prefix=".recertification-parent-") as staging: + parent_dir = _reconstructed_parent(release_dir, parent, Path(staging)) + try: + contract.validate_release_dir(parent_dir, artifact_root=artifact_root) + failures: list[str] = [] + except ReleaseContractError as exc: + failures = list(exc.failures) + evidence_failures: list[str] = [] + build = json.loads((parent_dir / "build_manifest.json").read_text()) + manifest = json.loads((parent_dir / "release_manifest.json").read_text()) + contract._check_us_stored_input_evidence( + parent_dir, + build_manifest=build, + release_manifest=manifest, + failures=evidence_failures, + evidence_tier=False, + ) + replaced = [line for line in failures if line in evidence_failures] + other = [line for line in failures if line not in evidence_failures] + return { + "status": PARENT_CONTRACT_PASSED + if not failures + else PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED, + "replaced_failures": replaced, + "other_failures": other, + } + + +# --------------------------------------------------------------------------- +# The loader replay under the tested wheels. +# --------------------------------------------------------------------------- + + +def _loader_values(frame, column): + import numpy as np + + values = frame[column].to_numpy() + if values.dtype == object: + return np.asarray([str(value) for value in values], dtype=object) + return values + + +def _h5_values(table, column): + import numpy as np + + values = table[column] + if values.dtype.kind == "S": + return np.asarray([value.decode() for value in values], dtype=object) + return values + + +def run_recertification_loader_replay( + candidate_h5: Path | str, + *, + require_wheels: bool = False, + compatibility_wheels: tuple[Path | str, ...] = (), +) -> dict: + """Load the H5 under the installed engine and say what it reads differently. + + The receipt this returns is replayed at every later gate, so it holds only + what is deterministic for the H5 and the installed runtime: + + 1. The four packages' installed sources, bound to the four wheels when + ``require_wheels`` (``source_enrichment._runtime_package_identities``). + 2. The stored-input classification (:func:`stored_column_changes`) and + the register digests. An unreviewed change refuses the replay. + 3. Both native loaders (country ``USSingleYearDataset``, wrapper + ``PolicyEngineUSDataset``) return every stored column the tested engine + defines with the H5's exact values, and every id, membership and weight + column too. + 4. When the H5 stores the native SPM role, its source comes from the + calculator wheel and a complete household's supplied role reaches Core + unchanged (``source_enrichment._check_native_input_precedence``). + + It runs no population simulation. Scoring the population is the drift + step, outside the receipt. + """ + + import inspect + + import h5py + import numpy as np + from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset + from policyengine_us.data import USSingleYearDataset + from policyengine_us.system import system + + from microcosm.data import source_enrichment, stored_inputs + + candidate_h5 = Path(candidate_h5) + packages = source_enrichment._runtime_package_identities( + compatibility_wheels, require_wheels=require_wheels + ) + tables = stored_inputs.h5_stored_tables(candidate_h5) + engine = stored_inputs.installed_us_engine() + changes = stored_column_changes( + tables, + engine_variables=engine.variables, + stored_input_register=stored_inputs.US_STORED_NON_VARIABLE_COLUMNS, + ) + failures = reviewed_column_change_failures(changes) + if failures: + raise ValueError(" ".join(failures)) + role = source_enrichment.ROLE_VARIABLE + stores_role = role in tables.get("person", ()) + source_paths = { + "country_loader": inspect.getsourcefile(USSingleYearDataset), + "wrapper_loader": inspect.getsourcefile(PolicyEngineUSDataset), + } + loaded_source_packages = { + "country_loader": "policyengine-us", + "wrapper_loader": "policyengine", + } + if stores_role: + variable = system.variables.get(role) + if ( + variable is None + or variable.entity.key != "person" + or (variable.value_type is not bool) + ): + raise ValueError(f"tested country must register {role} as a person bool") + source_paths["native_role"] = inspect.getsourcefile(type(variable)) + loaded_source_packages["native_role"] = "spm-calculator" + if require_wheels: + source_enrichment._check_loaded_source_ownership( + source_paths, loaded_source_packages + ) + country = USSingleYearDataset(file_path=str(candidate_h5)) + wrapper = PolicyEngineUSDataset( + name="compatibility_recertification", + description="Read-only compatibility recertification loader replay", + filepath=str(candidate_h5), + year=SCORING_PERIOD, + ) + checked = 0 + with h5py.File(candidate_h5, "r") as h5: + for entity in _ENTITIES: + table = h5[f"{entity}/table"] + names = [name for name in table.dtype.names if name != "index"] + structural = { + name + for name in names + if name in {f"{entity}_id", f"{entity}_weight"} + or ( + entity == "person" + and name.startswith("person_") + and name.endswith("_id") + ) + } + for column in names: + if column not in engine.variables and column not in structural: + continue + expected = _h5_values(table, column) + for loader, frame in ( + ("country", getattr(country, entity)), + ("wrapper", getattr(wrapper.data, entity)), + ): + if column not in frame.columns: + raise ValueError( + f"{loader} native loader dropped {entity}.{column}" + ) + observed = _loader_values(frame, column) + same = observed.shape == expected.shape and ( + np.array_equal(observed, expected) + if expected.dtype == object + else observed.dtype == expected.dtype + and observed.tobytes() == expected.tobytes() + ) + if not same: + raise ValueError( + f"{loader} native loader changed {entity}.{column}" + ) + checked += 1 + checks = [ + "country:every_stored_change_reviewed", + "country_and_wrapper:every_engine_variable_column_loaded_unchanged", + ] + if stores_role: + source_enrichment._check_native_input_precedence( + country, native_inputs={role: "person"} + ) + checks.append("country:complete_household_native_role_overrides_default") + columns = {column for names in tables.values() for column in names} + return { + "schema_version": RECEIPT_SCHEMA_VERSION, + "status": "passed", + "scope": "native_input_loading_only", + "external_package_publication": "not_attested", + "dataset_sha256": sha256_file(candidate_h5), + "runner_sha256": { + "compatibility_recertification": sha256_file(__file__), + "source_enrichment": sha256_file(source_enrichment.__file__), + }, + "packages": packages, + "loaded_source_packages": loaded_source_packages, + "loaded_source_sha256": { + key: sha256_file(path) for key, path in source_paths.items() + }, + "stored_inputs": { + "engine": engine.label, + "register_sha256": stored_inputs.register_sha256(), + "column_change_register_sha256": column_change_register_sha256(), + "registered_non_variables": sorted( + column + for column in columns + if stored_inputs.is_model_named(column) + and column not in engine.variables + and column in stored_inputs.US_STORED_NON_VARIABLE_COLUMNS + ), + "column_changes": changes, + }, + "columns_checked": checked, + "checks": checks, + } + + +# --------------------------------------------------------------------------- +# Drift: what is scored, and how two engines' scores combine. +# --------------------------------------------------------------------------- + +#: The baseline period and the SPM selection every score document records. +#: The selection must equal the parent's post-export scoring selection. +STATISTIC_SUM = "sum" +#: SOI amount rows: the positive part of the signed value (the release tool's +#: ``_signed_component``), so a refund-producing negative tax is not netted. +STATISTIC_POSITIVE_SUM = "positive_sum" +#: SOI return rows: units whose positive part is positive. +STATISTIC_POSITIVE_COUNT = "positive_count" +STATISTICS = (STATISTIC_SUM, STATISTIC_POSITIVE_SUM, STATISTIC_POSITIVE_COUNT) +#: Household batches (every measure is additive over households) or batches +#: of whole states (for the Medicaid state sums below). +BATCHING_HOUSEHOLD = "household" +BATCHING_STATE = "state" +#: Formulas that aggregate over the simulation's whole population. Copied from +#: ``tools/build_us_fiscal_refresh_release.py``'s +#: ``US_POPULATION_AGGREGATE_VARIABLES`` (a build test holds them equal). A +#: household-batch engine refuses to have computed any of them. +POPULATION_AGGREGATE_VARIABLES = ( + "household_income_decile", + "medicaid_slcsp_state_average_cost_index", + "medicaid_slcsp_state_denominator", + "spm_unit_income_decile", +) +#: The two of them that sum over one state (``sum_by_state`` in both +#: policyengine-us 2.2.1 and 2.60.4), so a batch of whole states computes them +#: exactly. A state-batch engine still refuses the deciles. +STATE_AGGREGATE_VARIABLES = ( + "medicaid_slcsp_state_average_cost_index", + "medicaid_slcsp_state_denominator", +) + + +@dataclass(frozen=True) +class DriftMetric: + """One scored quantity, defined over engine variables of one entity. + + ``variables`` are summed per unit of ``entity`` (a person variable over a + tax-unit metric is first summed to the tax unit), except ``combine = + "nonrefundable_ctc"``, which is ``max(min(ctc, ctc_limiting_tax_liability), + 0)`` as the release tool scores the SOI CTC rows. ``itemizers_only`` masks + by ``tax_unit_itemizes``. The weight is the H5's household weight of the + unit's household, so both engines weigh the same rows identically. + """ + + metric_id: str + label: str + entity: str + variables: tuple[str, ...] + statistic: str + unit: str + combine: str = "add" + itemizers_only: bool = False + batching: str = BATCHING_HOUSEHOLD + requirement_id: str | None = None + target_name: str | None = None + source_variable: str | None = None + + +#: The headline aggregates beside the critical rows. Each is a plain engine +#: total over the H5's household weights; none is a calibration target. +HEADLINE_METRICS = ( + DriftMetric( + "income_tax_net", + "federal income tax, net of refundable credits", + "tax_unit", + ("income_tax",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "ctc_total", + "Child Tax Credit, total", + "tax_unit", + ("ctc",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "eitc_total", + "Earned Income Tax Credit, total", + "tax_unit", + ("eitc",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "social_security_total", + "Social Security benefits", + "person", + ("social_security",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "snap_total", + "SNAP benefits", + "spm_unit", + ("snap",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "snap_units", + "SPM units receiving SNAP", + "spm_unit", + ("snap",), + STATISTIC_POSITIVE_COUNT, + "count", + ), + DriftMetric( + "ssi_total", + "SSI payments", + "person", + ("ssi",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "medicaid_enrollment", + "people enrolled in Medicaid", + "person", + ("medicaid_enrolled",), + STATISTIC_POSITIVE_COUNT, + "count", + batching=BATCHING_STATE, + ), + DriftMetric( + "medicaid_total", + "Medicaid benefit value", + "person", + ("medicaid",), + STATISTIC_SUM, + "usd", + batching=BATCHING_STATE, + ), + DriftMetric( + "state_income_tax_total", + "state income tax", + "tax_unit", + ("state_income_tax",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "adjusted_gross_income_total", + "adjusted gross income", + "tax_unit", + ("adjusted_gross_income",), + STATISTIC_SUM, + "usd", + ), +) + +#: The SOI ``source_variable`` names the release tool scores by identity, by +#: negative-part magnitude, or as ``max(value, 0)``; every other one by its +#: positive part (``tools/build_us_fiscal_refresh_release.py``'s +#: ``_signed_component``; a build test holds the two equal). +_SIGNED_IDENTITY = frozenset({"adjusted_gross_income", "rent_and_royalty_net_income"}) +_LOSS_SOURCES = frozenset( + { + "business_net_losses", + "capital_gains_losses", + "estate_losses", + "partnership_and_s_corp_losses", + "rent_and_royalty_net_losses", + } +) + + +def signed_component(values, source_variable: str): + """The part of a signed SOI value its row scores, as the release tool does.""" + + import numpy as np + + values = np.asarray(values, dtype=np.float64) + if source_variable in _SIGNED_IDENTITY: + return values + if source_variable == "capital_gains_losses": + return np.maximum(values, 0.0) + if source_variable.endswith("_losses") or source_variable in _LOSS_SOURCES: + return np.minimum(values, 0.0) * -1.0 + return np.maximum(values, 0.0) + + +def _soi_row_metric(target: Mapping, requirement_id: str | None) -> DriftMetric | None: + """A national, all-AGI, all-status SOI or SSA row as a metric, else ``None``. + + Only the rows whose release-tool materialization this module reproduces + are scored: the SOI slice on the national all-returns population (no state, + district, filing-status, AGI-band, child-count, positive-EITC or + taxable-only filter, no ledger filters other than the trivial ones), and + the plain-sum ``policyengine_variable`` rows. + """ + + metadata = _mapping(target.get("metadata")) + name = str(target.get("name") or "") + materializer = metadata.get("materializer") + base = metadata.get("base_variables") or metadata.get("base_variable") + if not isinstance(base, str) or not base: + return None + variables = tuple(part for part in base.split(",") if part) + mode = metadata.get("measure_mode", "sum") + if materializer == "policyengine_variable": + if mode != "sum" or any( + metadata.get(key) + for key in ( + "indicator_map_to", + "indicator_filter_variable", + "less_than", + "age_lower_bound", + "age_upper_bound", + "state_fips", + "congressional_district_geoid", + ) + ): + return None + return DriftMetric( + f"target:{name}", + str(metadata.get("diagnostic_target_label") or name), + "household_members", + variables, + STATISTIC_SUM, + str(metadata.get("ledger_measure_unit") or "usd"), + requirement_id=requirement_id, + target_name=name, + ) + if materializer != "irs_soi_slice": + return None + if ( + metadata.get("agi_lower_bound") != "-inf" + or metadata.get("agi_upper_bound") != "inf" + or metadata.get("filing_status") != "All" + or metadata.get("state_fips") + or metadata.get("congressional_district_geoid") + or metadata.get("taxable_only") == "true" + or metadata.get("requires_total_eitc_uprating") == "true" + or metadata.get("requires_total_soi_uprating") == "true" + or any( + key.startswith("ledger_filter_") + and key not in {"ledger_filter_filing_status", "ledger_filter_income_range"} + for key in metadata + ) + or metadata.get("ledger_filter_filing_status", "all") != "all" + or metadata.get("ledger_filter_income_range", "all") != "all" + ): + return None + source_variable = str(metadata.get("source_variable") or metadata.get("variable")) + if source_variable in {"count", "tax_filer_individual_count"}: + return None + combine = "add" + if source_variable == "ctc": + combine, variables = "nonrefundable_ctc", ("ctc", "ctc_limiting_tax_liability") + return DriftMetric( + f"target:{name}", + str(metadata.get("diagnostic_target_label") or name), + "tax_unit", + variables, + STATISTIC_POSITIVE_COUNT if mode == "indicator_sum" else STATISTIC_POSITIVE_SUM, + "count" if mode == "indicator_sum" else "usd", + combine=combine, + itemizers_only=metadata.get("itemized_only") == "true", + requirement_id=requirement_id, + target_name=name, + source_variable=source_variable, + ) + + +def drift_metric_definitions(calibration_diagnostics: Mapping) -> dict: + """Every scored metric, derived from the parent's diagnostics and this module. + + Returns ``{"metrics": [...], "unscored_critical_rows": [...]}``. The + metrics are :data:`HEADLINE_METRICS` plus one per national row of the 18 + exact critical requirements and of the SOI Table 1.4 dollar blanket that + :func:`_soi_row_metric` reproduces. A critical row it cannot reproduce is + listed, and :func:`combine_drift` refuses a drift that leaves one of the 18 + unscored. + """ + + from microcosm.data.us_critical_targets import ( + US_EXACT_CRITICAL_TARGET_FIT_REQUIREMENTS, + US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT, + is_congressional_district_target, + ) + + metrics = list(HEADLINE_METRICS) + unscored = [] + seen = set() + for target in calibration_diagnostics.get("targets") or (): + if not isinstance(target, Mapping): + continue + name = str(target.get("name") or "") + metadata = _mapping(target.get("metadata")) + if not name or name in seen or is_congressional_district_target(name, metadata): + continue + family = name.split(".", 1)[0] + role = str(metadata.get("target_role") or "") + requirement = next( + ( + item + for item in ( + *US_EXACT_CRITICAL_TARGET_FIT_REQUIREMENTS, + US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT, + ) + if item.matches(name=name, family=family, target_role=role) + ), + None, + ) + if requirement is None: + continue + seen.add(name) + metric = _soi_row_metric(target, requirement.requirement_id) + if metric is None: + unscored.append( + {"target_name": name, "requirement_id": requirement.requirement_id} + ) + continue + metrics.append(metric) + return { + "metrics": [asdict(metric) for metric in metrics], + "unscored_critical_rows": unscored, + } + + +def _exact_requirement_ids() -> tuple[str, ...]: + from microcosm.data.us_critical_targets import ( + US_EXACT_CRITICAL_TARGET_FIT_REQUIREMENTS, + ) + + return tuple( + item.requirement_id for item in US_EXACT_CRITICAL_TARGET_FIT_REQUIREMENTS + ) + + +def _critical_tolerances() -> dict[str, float]: + from microcosm.data.us_critical_targets import US_CRITICAL_TARGET_FIT_REQUIREMENTS + + return { + item.requirement_id: float(item.max_abs_relative_error) + for item in US_CRITICAL_TARGET_FIT_REQUIREMENTS + } + + +#: Score variants. ``as_stored`` scores the H5 as published; the counterfactual +#: withholds the columns the tested engine newly reads (``column_changes.read``), +#: so ``as_stored - without_newly_read`` is their effect and +#: ``without_newly_read - baseline`` the rest of the engine change. +VARIANT_AS_STORED = "as_stored" +VARIANT_WITHOUT_NEWLY_READ = "without_newly_read_inputs" + + +def _score_document_failures( + scores: Mapping, *, dataset_sha256: str, definitions: Mapping, label: str +) -> list[str]: + """Shape and binding checks for one embedded score document.""" + + failures = [] + if ( + scores.get("schema_version") != SCORES_SCHEMA_VERSION + or scores.get("kind") != SCORES_KIND + ): + failures.append( + f"{label} scores must be a schema-{SCORES_SCHEMA_VERSION} {SCORES_KIND}" + ) + if _mapping(scores.get("dataset")).get("sha256") != dataset_sha256: + failures.append(f"{label} scores must score the recertified H5") + if scores.get("metric_definitions_sha256") != canonical_sha256(definitions): + failures.append( + f"{label} scores used other metric definitions than this release's" + ) + values = _mapping(scores.get("metrics")) + expected = {metric["metric_id"] for metric in definitions["metrics"]} + if set(values) != expected: + failures.append(f"{label} scores must score exactly the defined metrics") + for metric_id, raw in values.items(): + value = _mapping(raw).get("value") + if ( + not isinstance(value, int | float) + or isinstance(value, bool) + or not math.isfinite(value) + ): + failures.append(f"{label} metric {metric_id!r} has no finite value") + code = _mapping(scores.get("code")) + if set(_mapping(code.get("source_files_sha256"))) != set(PRODUCER_SOURCE_FILES): + failures.append(f"{label} scores must record the producer source inventory") + return failures + + +def _relative(value: float, reference: float | None) -> float | None: + if reference is None or reference == 0: + return None + return (value - reference) / abs(reference) + + +def combine_drift( + *, + calibration_diagnostics: Mapping, + dataset_sha256: str, + baseline: Mapping, + tested: Mapping, + counterfactual: Mapping | None, + baseline_engine: Mapping, + tested_packages: Mapping, + newly_read_columns: Sequence[str], +) -> dict: + """Combine score documents into the drift document, or raise ``ValueError``. + + ``baseline`` must have been scored under the parent's built-with + policyengine-us and policyengine-core (``baseline_engine``); ``tested`` + under exactly ``tested_packages`` (every package's version and installed + source digest, as the loader replay recorded them). ``counterfactual``, if + given, must be the tested runtime with exactly ``newly_read_columns`` + withheld, and is required whenever there are any. Every row recomputes + from the embedded documents. + """ + + definitions = drift_metric_definitions(calibration_diagnostics) + failures = [] + exact = set(_exact_requirement_ids()) + scored_exact = { + metric["requirement_id"] + for metric in definitions["metrics"] + if metric["requirement_id"] in exact + } + if scored_exact != exact: + failures.append( + "drift must score every exact critical row; unscored: " + f"{sorted(exact - scored_exact)}" + ) + sides = [ + ("baseline", baseline, VARIANT_AS_STORED), + ("tested", tested, VARIANT_AS_STORED), + ] + if newly_read_columns: + if counterfactual is None: + failures.append( + "the tested engine newly reads stored columns, so the drift needs " + f"the {VARIANT_WITHOUT_NEWLY_READ} counterfactual score" + ) + else: + sides.append(("counterfactual", counterfactual, VARIANT_WITHOUT_NEWLY_READ)) + elif counterfactual is not None: + failures.append("a counterfactual score needs newly read columns to withhold") + for label, scores, variant in sides: + failures.extend( + _score_document_failures( + scores, + dataset_sha256=dataset_sha256, + definitions=definitions, + label=label, + ) + ) + recorded_variant = _mapping(scores.get("variant")) + withheld = ( + sorted(newly_read_columns) if variant == VARIANT_WITHOUT_NEWLY_READ else [] + ) + if recorded_variant != {"id": variant, "withheld_columns": withheld}: + failures.append(f"{label} scores must be the {variant!r} variant") + baseline_packages = _mapping(_mapping(baseline.get("runtime")).get("packages")) + for package, field in (("policyengine-us", "model"), ("policyengine-core", "core")): + want = _mapping(baseline_engine.get(field)).get("version") + if _mapping(baseline_packages.get(package)).get("version") != want: + failures.append( + f"baseline scores must run the parent's built-with {package} {want}" + ) + for label, scores in (("tested", tested), ("counterfactual", counterfactual)): + if scores is None: + continue + observed = _mapping(_mapping(scores.get("runtime")).get("packages")) + for package in COMPATIBILITY_PACKAGES: + want = _mapping(tested_packages.get(package)) + got = _mapping(observed.get(package)) + if (got.get("version"), got.get("source_sha256")) != ( + want.get("version"), + want.get("source_sha256"), + ): + failures.append( + f"{label} scores must run the certified {package} runtime" + ) + if failures: + raise ValueError(" ".join(failures)) + targets = { + str(target.get("name")): target + for target in calibration_diagnostics.get("targets") or () + if isinstance(target, Mapping) + } + tolerances = _critical_tolerances() + rows = [] + for metric in definitions["metrics"]: + metric_id = metric["metric_id"] + base = float(baseline["metrics"][metric_id]["value"]) + new = float(tested["metrics"][metric_id]["value"]) + row = { + "metric_id": metric_id, + "label": metric["label"], + "unit": metric["unit"], + "requirement_id": metric["requirement_id"], + "baseline": base, + "tested": new, + "difference": new - base, + "relative_difference": _relative(new, base), + } + target = targets.get(metric["target_name"] or "") + if target is not None: + value = target.get("target") + build = target.get("final_estimate") + row["target"] = value + row["build_estimate"] = build + if isinstance(value, int | float) and value: + row["target_relative_error"] = { + "baseline": _relative(base, value), + "tested": _relative(new, value), + } + tolerance = tolerances.get(metric["requirement_id"] or "") + if tolerance is not None: + row["tolerance"] = tolerance + row["within_tolerance"] = { + "baseline": abs(_relative(base, value)) <= tolerance, + "tested": abs(_relative(new, value)) <= tolerance, + } + if counterfactual is not None: + withheld = float(counterfactual["metrics"][metric_id]["value"]) + row[VARIANT_WITHOUT_NEWLY_READ] = withheld + row["newly_read_inputs_effect"] = new - withheld + row["other_engine_changes_effect"] = withheld - base + rows.append(row) + return { + "schema_version": DRIFT_SCHEMA_VERSION, + "kind": DRIFT_KIND, + "dataset_sha256": dataset_sha256, + "metric_definitions_sha256": canonical_sha256(definitions), + "unscored_critical_rows": definitions["unscored_critical_rows"], + "newly_read_columns": sorted(newly_read_columns), + "rows": rows, + "filing_status": { + "baseline": baseline.get("filing_status"), + "tested": tested.get("filing_status"), + }, + "tax_unit_roles": { + "baseline": baseline.get("tax_unit_roles"), + "tested": tested.get("tax_unit_roles"), + }, + "scores": { + "baseline": dict(baseline), + "tested": dict(tested), + **( + {} + if counterfactual is None + else {"counterfactual": dict(counterfactual)} + ), + }, + } + + +#: The rows the drift summary names, in order: the headline programmes Max +#: asked about, then the critical rows that carry them. +SUMMARY_METRICS = ( + "income_tax_net", + "eitc_total", + "ctc_total", + "social_security_total", + "snap_total", + "medicaid_total", + "medicaid_enrollment", +) + + +def drift_summary(drift: Mapping) -> dict: + """The headline the report carries beside the claim, recomputed from the rows.""" + + rows = {row["metric_id"]: row for row in drift.get("rows", ())} + critical = [row for row in drift.get("rows", ()) if row.get("requirement_id")] + exact = set(_exact_requirement_ids()) + exact_rows = [row for row in critical if row["requirement_id"] in exact] + largest = max( + (row for row in exact_rows if row["relative_difference"] is not None), + key=lambda row: abs(row["relative_difference"]), + default=None, + ) + + def leaving_tolerance(rows): + return sorted( + row["metric_id"] + for row in rows + if _mapping(row.get("within_tolerance")).get("baseline") is True + and _mapping(row.get("within_tolerance")).get("tested") is False + ) + + return { + "headline": { + metric_id: { + key: rows[metric_id].get(key) + for key in ( + "baseline", + "tested", + "relative_difference", + "newly_read_inputs_effect", + ) + if key in rows[metric_id] + } + for metric_id in SUMMARY_METRICS + if metric_id in rows + }, + "exact_critical_rows": len(exact_rows), + "largest_exact_critical_move": None + if largest is None + else { + "metric_id": largest["metric_id"], + "requirement_id": largest["requirement_id"], + "relative_difference": largest["relative_difference"], + }, + "critical_rows_leaving_tolerance": leaving_tolerance(critical), + "critical_rows_outside_tolerance_under_tested": sorted( + row["metric_id"] + for row in critical + if _mapping(row.get("within_tolerance")).get("tested") is False + ), + } + + +def drift_from_scores( + drift: Mapping, + *, + calibration_diagnostics: Mapping, + baseline_engine: Mapping, + tested_packages: Mapping, + newly_read_columns: Sequence[str], +) -> dict: + """Recompute a drift document from the score documents it embeds.""" + + scores = _mapping(drift.get("scores")) + return combine_drift( + calibration_diagnostics=calibration_diagnostics, + dataset_sha256=str(drift.get("dataset_sha256")), + baseline=_mapping(scores.get("baseline")), + tested=_mapping(scores.get("tested")), + counterfactual=scores.get("counterfactual"), + baseline_engine=baseline_engine, + tested_packages=tested_packages, + newly_read_columns=newly_read_columns, + ) + + +# --------------------------------------------------------------------------- +# The claim. +# --------------------------------------------------------------------------- + + +def claim_entries( + parent_manifest: Mapping, + *, + tested_packages: Mapping, + specifier: str, + declared_by: str, +) -> tuple[list[dict], list[dict], dict]: + """The child's model and Core compatibility lists and its claims block. + + Every parent entry is kept, so consumers on the built-with version keep + their certification. The model gains one ``publisher_claim`` entry, which + must contain the tested version and pass the source-enrichment claim + guards (bounded above and below; ``compatibility_claim_entry``). Core + gains an exact pin of the tested Core as a ``publisher_claim``, unless a + parent entry already covers it. + + Raises: + ValueError: The claim is malformed, does not cover the tested version, + or the parent already covers the tested model version. + """ + + from packaging.specifiers import SpecifierSet + from packaging.version import Version + + from microcosm.data.source_enrichment import compatibility_claim_entry + + model_version = _mapping(tested_packages.get("policyengine-us")).get("version") + core_version = _mapping(tested_packages.get("policyengine-core")).get("version") + models = [ + dict(entry) for entry in parent_manifest.get("compatible_model_packages") or () + ] + cores = [ + dict(entry) for entry in parent_manifest.get("compatible_core_packages") or () + ] + + def covered(entries, name, version): + return any( + entry.get("name") == name + and Version(version) in SpecifierSet(entry["specifier"]) + for entry in entries + ) + + if covered(models, "policyengine-us", model_version): + raise ValueError( + f"the parent already certifies policyengine-us {model_version}; " + "there is nothing to recertify" + ) + model = compatibility_claim_entry( + specifier, + package="policyengine-us", + version=model_version, + declared_by=declared_by, + ) + claims = {"model": model} + if not covered(cores, "policyengine-core", core_version): + claims["core"] = { + "name": "policyengine-core", + "specifier": f"=={core_version}", + "basis": PUBLISHER_CLAIM_BASIS, + "declared_by": declared_by, + } + cores.append(claims["core"]) + models.append(model) + return models, cores, claims + + +def claim_measurement( + claims: Mapping, + *, + tested_version: str, + baseline_version: str, + drift: Mapping, + drift_sha256: str, +) -> dict: + """What the claim rests on: one measured version, its baseline, the drift.""" + + return { + "model": { + "specifier": claims["model"]["specifier"], + "measured_versions": [tested_version], + "baseline_version": baseline_version, + "drift": { + "filename": DRIFT_FILE, + "sha256": drift_sha256, + "summary": drift_summary(drift), + }, + } + } + + +# --------------------------------------------------------------------------- +# The validator. +# --------------------------------------------------------------------------- + + +def _check_producer(code: Mapping, failures: list[str], *, label: str) -> None: + from microcosm.data.source_enrichment import _check_producer_source_identity + + try: + _check_producer_source_identity(code, inventory=PRODUCER_SOURCE_FILES) + except (OSError, ValueError) as exc: + failures.append(f"{label} producer source identity failed: {exc}") + + +def validate_recertification_candidate( + release_dir: Path | str, + *, + artifact_root: Path | str | None, + require_compatibility: bool = False, + compatibility_wheels: tuple[Path | str, ...] = (), +) -> dict: + """Check a recertification child; return its report or raise. + + Checks, in order: the carried parent manifests and the parent block; every + parent artifact republished byte for byte with the child's revision; the + parent's contract replayed on the carried bytes; the child's own manifest + under the general release-manifest checks; and, once compatibility has + passed, the loader replay, the claim and the drift, each recomputed. With + ``require_compatibility`` (certification output, publish preflight and + publication) a pending child is refused, the loader replay must bind the + four wheels, and every producer is authenticated against this checkout. + """ + + from microcosm.data.contract import _check_release_manifest + + release_dir = Path(release_dir) + artifact_root = Path(artifact_root) if artifact_root is not None else None + failures: list[str] = [] + manifest = _json(release_dir / "release_manifest.json", failures) + build = _json(release_dir / "build_manifest.json", failures) + report = _json(release_dir / REPORT_FILE, failures) + release_id = release_dir.name + parent = _mapping(report.get("parent")) + parent_id = parent.get("build_id") + if manifest.get("release_type") != RECERTIFICATION_RELEASE_TYPE: + failures.append( + f"release_manifest.json must declare release_type={RECERTIFICATION_RELEASE_TYPE}" + ) + if ( + report.get("schema_version") != REPORT_SCHEMA_VERSION + or report.get("release_type") != RECERTIFICATION_RELEASE_TYPE + ): + failures.append( + f"{REPORT_FILE} must be a schema-{REPORT_SCHEMA_VERSION} report" + ) + if ( + not isinstance(parent_id, str) + or release_id == parent_id + or not release_id.startswith(US_RELEASE_PREFIX) + ): + failures.append("a recertification needs a NEW US release id, not its parent's") + if ( + _mapping(manifest.get("build")).get("build_id") != release_id + or build.get("build_id") != release_id + ): + failures.append( + "both manifests' build_id must name the child release directory" + ) + if build.get("release_type") != RECERTIFICATION_RELEASE_TYPE: + failures.append( + f"build_manifest.json must declare {RECERTIFICATION_RELEASE_TYPE}" + ) + if report.get("transformation") != "none": + failures.append(f"{REPORT_FILE} must declare transformation 'none'") + if failures: + raise ReleaseContractError(release_dir, failures) + + # 1. The carried parent manifests are the identity the report names. + carried_manifest = release_dir / PARENT_RELEASE_MANIFEST_FILE + carried_build = release_dir / PARENT_BUILD_MANIFEST_FILE + for path, field in ( + (carried_manifest, "release_manifest_sha256"), + (carried_build, "build_manifest_sha256"), + ): + if not path.is_file() or sha256_file(path) != parent.get(field): + failures.append(f"{path.name} must be the parent bytes the report pins") + if failures: + raise ReleaseContractError(release_dir, failures) + with tempfile.TemporaryDirectory(prefix=".recertification-describe-") as staging: + try: + described = _describe_carried_parent( + release_dir, parent, artifact_root, Path(staging) + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + raise ReleaseContractError( + release_dir, + [f"carried parent does not describe a supported parent: {exc}"], + ) from exc + if described != dict(parent): + failures.append( + f"{REPORT_FILE} parent block must equal what the carried parent manifest " + "and files describe" + ) + parent_manifest = json.loads(carried_manifest.read_text()) + + # 2. Every parent artifact republished, bytes unchanged. + dataset = _mapping(report.get("dataset")) + if dataset != parent.get("dataset"): + failures.append( + f"{REPORT_FILE} dataset must be the parent's microdata, unchanged" + ) + child_artifacts = _mapping(manifest.get("artifacts")) + expected_artifacts = {} + for key, raw in _mapping(parent_manifest.get("artifacts")).items(): + entry = dict(_mapping(raw)) + entry["revision"] = release_id + expected_artifacts[key] = entry + for key, name in CHILD_ARTIFACT_KEYS.items(): + expected_artifacts[key] = { + "kind": "evidence", + "path": name, + "repo_id": REPO_ID, + "revision": release_id, + "sha256": sha256_file(release_dir / name) + if (release_dir / name).is_file() + else None, + } + compatibility = _mapping(report.get("compatibility")) + passed = compatibility.get("status") == "passed" + if passed: + for key, name in CERTIFIED_ARTIFACT_KEYS.items(): + expected_artifacts[key] = { + "kind": "diagnostics", + "path": name, + "repo_id": REPO_ID, + "revision": release_id, + "sha256": sha256_file(release_dir / name) + if (release_dir / name).is_file() + else None, + } + if dict(child_artifacts) != expected_artifacts: + failures.append( + "release manifest artifacts must be the parent's, re-pinned to the child " + "tag, plus the child's own evidence, each hashing to its file" + ) + for name, sha in _mapping(parent.get("release_files")).items(): + if not (release_dir / name).is_file() or sha256_file(release_dir / name) != sha: + failures.append(f"carried parent file {name} must keep the parent bytes") + if artifact_root is None: + failures.append( + "a recertification requires artifact_root for its root artifacts" + ) + else: + for path, sha in _mapping(parent.get("root_artifacts")).items(): + local = artifact_root / path + if not local.is_file() or sha256_file(local) != sha: + failures.append(f"root artifact {path} must keep the parent bytes") + if (release_dir / path).exists() or (release_dir / path).is_symlink(): + failures.append( + f"root artifact {path} must live only in artifact_root, never in " + "the release directory" + ) + expected_files = { + *CHILD_ARTIFACT_KEYS.values(), + "release_manifest.json", + *_mapping(parent.get("release_files")), + *(CERTIFIED_ARTIFACT_KEYS.values() if passed else ()), + } + present = {path.name for path in release_dir.iterdir()} + if present != expected_files: + failures.append( + "release directory must hold exactly the carried parent files and the " + f"child's own; unexpected {sorted(present - expected_files)}, missing " + f"{sorted(expected_files - present)}" + ) + + # 3. The parent passes today's contract on the carried bytes. + parent_contract = replay_parent_contract( + release_dir, parent, artifact_root=artifact_root + ) + if parent_contract["other_failures"]: + failures.append( + "the parent fails the release contract on its carried bytes: " + + " ".join(parent_contract["other_failures"]) + ) + if report.get("parent_contract") != parent_contract: + failures.append( + f"{REPORT_FILE} parent_contract must equal the replayed contract" + ) + + # 4. The child's manifest is a release manifest; its fields are the parent's. + parent_build = _mapping(parent_manifest.get("build")) + for field in ("built_with_model_package", "built_with_core_package"): + if _mapping(_mapping(manifest.get("build")).get(field)) != _mapping( + parent_build.get(field) + ): + failures.append( + f"build.{field} must stay the parent's: the H5 was built with it" + ) + if _mapping(manifest.get("build")).get("recertified_from") != parent_id: + failures.append("build.recertified_from must name the parent") + for field in ( + "schema_version", + "default_datasets", + "calibration_diagnostics", + "dataset_role", + ): + if manifest.get(field) != parent_manifest.get(field): + failures.append(f"release manifest {field} must stay the parent's") + _check_release_manifest(manifest, release_id, failures) + calibration = _mapping(build.get("calibration")) + diagnostics_sha = _mapping(parent.get("release_files")).get( + "calibration_diagnostics.json" + ) + if calibration != { + "mode": "inherited", + "parent_build_id": parent_id, + "diagnostics_sha256": diagnostics_sha, + }: + failures.append( + "build calibration must explicitly inherit the parent's diagnostics" + ) + if _mapping(build.get("dataset")) != dict(dataset): + failures.append("build manifest dataset must be the parent's microdata") + + # 5. Compatibility. + if compatibility.get("status") == "pending": + if require_compatibility: + failures.append( + "recertification compatibility is pending; certify it under the " + "tested wheels" + ) + for field in ("compatible_model_packages", "compatible_core_packages"): + if manifest.get(field) != parent_manifest.get(field): + failures.append( + f"a pending recertification must carry the parent's {field}" + ) + if set(compatibility) != {"status"}: + failures.append("a pending recertification records no claim or measurement") + elif passed: + _check_certified_compatibility( + release_dir, + manifest=manifest, + parent_manifest=parent_manifest, + report=report, + parent_contract=parent_contract, + compatibility=compatibility, + candidate=None + if artifact_root is None + else artifact_root / str(dataset.get("filename")), + require_wheels=require_compatibility, + compatibility_wheels=compatibility_wheels, + failures=failures, + ) + else: + failures.append( + "recertification compatibility status must be pending or passed" + ) + if require_compatibility: + _check_producer(_mapping(build.get("code")), failures, label="release") + if passed: + drift = _json(release_dir / DRIFT_FILE, []) + for label, scores in _mapping(drift.get("scores")).items(): + _check_producer( + _mapping(_mapping(scores).get("code")), + failures, + label=f"{label} scores", + ) + if failures: + raise ReleaseContractError(release_dir, failures) + return report + + +def _describe_carried_parent( + release_dir: Path, parent: Mapping, artifact_root: Path | None, staging: Path +) -> dict: + """:func:`describe_parent` on the parent rebuilt from the carried bytes.""" + + directory = _reconstructed_parent(release_dir, parent, staging) + if artifact_root is None: + raise ValueError("artifact_root is required") + return describe_parent(directory, artifact_root) + + +def _check_certified_compatibility( + release_dir: Path, + *, + manifest: Mapping, + parent_manifest: Mapping, + report: Mapping, + parent_contract: Mapping, + compatibility: Mapping, + candidate: Path | None, + require_wheels: bool, + compatibility_wheels: tuple[Path | str, ...], + failures: list[str], +) -> None: + from microcosm.data.stored_inputs import StoredInputRefusalError, register_sha256 + + receipt_path = release_dir / RECEIPT_FILE + drift_path = release_dir / DRIFT_FILE + if compatibility.get("filename") != RECEIPT_FILE or ( + not receipt_path.is_file() + or compatibility.get("sha256") != sha256_file(receipt_path) + ): + failures.append("compatibility must hash-bind the loader replay receipt") + return + receipt = _json(receipt_path, failures) + if candidate is None or not candidate.is_file(): + failures.append("the recertified H5 must be present to replay the loader") + return + try: + actual = run_recertification_loader_replay( + candidate, + require_wheels=require_wheels, + compatibility_wheels=compatibility_wheels, + ) + except ( + StoredInputRefusalError, + ValueError, + OSError, + ImportError, + KeyError, + TypeError, + ) as exc: + failures.append(f"loader replay failed: {exc}") + return + if receipt != actual: + failures.append( + "loader receipt differs from the replay under the installed runtime" + ) + return + tested_packages = _mapping(actual.get("packages")) + changes = _mapping(_mapping(actual.get("stored_inputs")).get("column_changes")) + if compatibility.get("stored_column_changes") != recorded_column_changes(changes): + failures.append( + "compatibility.stored_column_changes must be the replay's changes with " + "their reviewed reasons" + ) + parent_build = _mapping(parent_manifest.get("build")) + baseline_engine = { + "model": _mapping(parent_build.get("built_with_model_package")), + "core": _mapping(parent_build.get("built_with_core_package")), + } + claims = _mapping(compatibility.get("publisher_claims")) + model_claim = _mapping(claims.get("model")) + try: + models, cores, expected_claims = claim_entries( + parent_manifest, + tested_packages=tested_packages, + specifier=model_claim.get("specifier"), + declared_by=model_claim.get("declared_by"), + ) + except (ValueError, TypeError) as exc: + failures.append(f"publisher compatibility claim: {exc}") + return + if dict(claims) != expected_claims: + failures.append( + "compatibility.publisher_claims must be exactly the validated claims" + ) + if manifest.get("compatible_model_packages") != models: + failures.append( + "compatible_model_packages must be the parent's entries plus the declared claim" + ) + if manifest.get("compatible_core_packages") != cores: + failures.append( + "compatible_core_packages must be the parent's entries plus the tested Core" + ) + if not drift_path.is_file(): + failures.append(f"a certified recertification must deliver {DRIFT_FILE}") + return + drift = _json(drift_path, failures) + diagnostics = _json(release_dir / "calibration_diagnostics.json", failures) + newly_read = [item["column"] for item in changes.get(CHANGE_READ, ())] + try: + expected_drift = drift_from_scores( + drift, + calibration_diagnostics=diagnostics, + baseline_engine=baseline_engine, + tested_packages=tested_packages, + newly_read_columns=newly_read, + ) + except (ValueError, KeyError, TypeError) as exc: + failures.append(f"drift does not recompute from its scores: {exc}") + return + if drift != expected_drift: + failures.append(f"{DRIFT_FILE} differs from the drift its scores recompute") + if drift.get("dataset_sha256") != _mapping(report.get("dataset")).get("sha256"): + failures.append("drift must score the recertified H5") + if parent_contract.get("status") == PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED: + verdict = _mapping( + _mapping( + _mapping(_mapping(drift.get("scores")).get("baseline")).get("engine") + ).get("stored_input_verdict") + ) + if ( + verdict.get("passed") is not True + or verdict.get("register_sha256") != register_sha256() + ): + failures.append( + "the parent carries no stored-input evidence, so the baseline scores " + "must record a passing stored-input verdict under the built-with " + "engine against the current register" + ) + expected_measurement = claim_measurement( + expected_claims, + tested_version=str( + _mapping(tested_packages.get("policyengine-us")).get("version") + ), + baseline_version=str(baseline_engine["model"].get("version")), + drift=expected_drift, + drift_sha256=sha256_file(drift_path), + ) + if compatibility.get("claim_measurement") != expected_measurement: + failures.append( + "compatibility.claim_measurement must name the measured version, its " + "baseline and the drift summary its scores recompute" + ) + + +# --------------------------------------------------------------------------- +# Certification. +# --------------------------------------------------------------------------- + + +def certify_recertification( + release_dir: Path | str, + output_dir: Path | str, + *, + artifact_root: Path | str, + compatibility_wheels: tuple[Path | str, ...], + baseline_scores: Path | str, + tested_scores: Path | str, + counterfactual_scores: Path | str | None, + compatible_model_specifier: str, + compatibility_claim_declared_by: str, +) -> Path: + """Write a certified child next to the pending candidate, or raise. + + The loader replay runs under the installed runtime with the four wheels; + the three score documents must be the ones the drift recomputes from. The + claim is declared here and recorded as the declarer's, with the drift + summary beside it. The candidate and its H5 are left unchanged. + """ + + from microcosm.data.source_enrichment import ( + check_compatibility_claim_declarer, + parse_compatibility_claim_requirement, + ) + + release_dir, output_dir = Path(release_dir), Path(output_dir) + if output_dir.exists() or output_dir.name != release_dir.name: + raise ValueError( + "output_dir must be new and keep the candidate release id as its basename" + ) + specifier = parse_compatibility_claim_requirement( + compatible_model_specifier, package="policyengine-us" + ) + check_compatibility_claim_declarer(compatibility_claim_declared_by) + report = validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + if report["compatibility"] != {"status": "pending"}: + raise ValueError("certify a pending candidate, not a certified bundle") + candidate = Path(artifact_root) / report["dataset"]["filename"] + receipt = run_recertification_loader_replay( + candidate, require_wheels=True, compatibility_wheels=compatibility_wheels + ) + tested_packages = receipt["packages"] + parent_manifest = json.loads( + (release_dir / PARENT_RELEASE_MANIFEST_FILE).read_text() + ) + parent_build = parent_manifest["build"] + models, cores, claims = claim_entries( + parent_manifest, + tested_packages=tested_packages, + specifier=specifier, + declared_by=compatibility_claim_declared_by, + ) + changes = receipt["stored_inputs"]["column_changes"] + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + drift = combine_drift( + calibration_diagnostics=diagnostics, + dataset_sha256=report["dataset"]["sha256"], + baseline=json.loads(Path(baseline_scores).read_text()), + tested=json.loads(Path(tested_scores).read_text()), + counterfactual=None + if counterfactual_scores is None + else json.loads(Path(counterfactual_scores).read_text()), + baseline_engine={ + "model": parent_build["built_with_model_package"], + "core": parent_build["built_with_core_package"], + }, + tested_packages=tested_packages, + newly_read_columns=[item["column"] for item in changes[CHANGE_READ]], + ) + output_dir.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".recertification-", dir=output_dir.parent + ) as staging: + staged = Path(staging) / release_dir.name + shutil.copytree(release_dir, staged) + for path in staged.iterdir(): + path.chmod(0o600) + write_json(staged / RECEIPT_FILE, receipt) + write_json(staged / DRIFT_FILE, drift) + report["compatibility"] = { + "status": "passed", + "filename": RECEIPT_FILE, + "sha256": sha256_file(staged / RECEIPT_FILE), + "stored_column_changes": recorded_column_changes(changes), + "publisher_claims": claims, + "claim_measurement": claim_measurement( + claims, + tested_version=tested_packages["policyengine-us"]["version"], + baseline_version=parent_build["built_with_model_package"]["version"], + drift=drift, + drift_sha256=sha256_file(staged / DRIFT_FILE), + ), + } + write_json(staged / REPORT_FILE, report) + manifest = json.loads((staged / "release_manifest.json").read_text()) + manifest["compatible_model_packages"] = models + manifest["compatible_core_packages"] = cores + for key, name in {**CHILD_ARTIFACT_KEYS, **CERTIFIED_ARTIFACT_KEYS}.items(): + manifest["artifacts"][key] = { + "kind": "diagnostics" if key in CERTIFIED_ARTIFACT_KEYS else "evidence", + "path": name, + "repo_id": REPO_ID, + "revision": release_dir.name, + "sha256": sha256_file(staged / name), + } + write_json(staged / "release_manifest.json", manifest) + validate_recertification_candidate( + staged, + artifact_root=artifact_root, + require_compatibility=True, + compatibility_wheels=compatibility_wheels, + ) + for path in staged.iterdir(): + path.chmod(0o400) + staged.rename(output_dir) + return output_dir + + +# --------------------------------------------------------------------------- +# Publication: the parent must be published with the carried bytes. +# --------------------------------------------------------------------------- + + +def require_parent_publication( + api: object, release_dir: Path | str, *, repo_id: str +) -> None: + """Refuse unless the parent tag on the Hub serves the carried manifests. + + Runs after the publisher constructs its Hub client and before any upload. + The parent's ``release_manifest.json`` pins every parent artifact, so equal + bytes at the parent's immutable tag mean the lineage the child carries is + the published one. Absence and difference both refuse; a transient Hub + failure propagates. + """ + + from microcosm.data.release import _is_absent_entry_error, _is_absent_revision_error + + release_dir = Path(release_dir) + report = json.loads((release_dir / REPORT_FILE).read_text()) + parent = _mapping(report.get("parent")) + parent_id, revision = parent.get("build_id"), parent.get("revision") + if parent.get("repo_id") != repo_id: + raise ValueError( + f"the recertification's parent lives in {parent.get('repo_id')!r}, " + f"not the publication repo {repo_id!r}" + ) + for carried, remote_name in ( + (PARENT_RELEASE_MANIFEST_FILE, "release_manifest.json"), + (PARENT_BUILD_MANIFEST_FILE, "build_manifest.json"), + ): + path_in_repo = f"releases/{parent_id}/{remote_name}" + try: + remote = Path( + api.hf_hub_download( + repo_id=repo_id, + filename=path_in_repo, + repo_type="dataset", + revision=revision, + ) + ).read_bytes() + except Exception as exc: + if not (_is_absent_entry_error(exc) or _is_absent_revision_error(exc)): + raise + raise ValueError( + f"parent {parent_id!r} is not published: {path_in_repo} is absent " + f"at tag {revision!r} in {repo_id} ({exc}). A recertification " + "republishes only a published parent." + ) from exc + if remote != (release_dir / carried).read_bytes(): + raise ValueError( + f"{path_in_repo} at tag {revision!r} differs from the carried " + f"{carried}; refusing to publish a child of a parent the Hub does " + "not serve." + ) + + +# --------------------------------------------------------------------------- +# CLI. +# --------------------------------------------------------------------------- + + +def recorded_claim_measurement(release_dir: Path | str) -> dict: + """The claim measurement a certified child records, or ``{}``; a report path.""" + + try: + report = json.loads((Path(release_dir) / REPORT_FILE).read_text()) + except (OSError, ValueError): + return {} + record = _mapping(_mapping(report).get("compatibility")).get("claim_measurement") + return dict(record) if isinstance(record, Mapping) else {} + + +def main(argv: list[str] | None = None) -> int: + import argparse + import sys + + parser = argparse.ArgumentParser( + prog="python -m microcosm.data.compatibility_recertification", + description=__doc__.split("\n\n")[0], + ) + parser.add_argument("--release-dir", required=True, type=Path) + parser.add_argument("--artifact-root", required=True, type=Path) + parser.add_argument("--certify", action="store_true") + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--require-compatibility", action="store_true") + parser.add_argument("--compatibility-wheel", action="append", type=Path, default=[]) + parser.add_argument("--baseline-scores", type=Path) + parser.add_argument("--tested-scores", type=Path) + parser.add_argument("--counterfactual-scores", type=Path) + parser.add_argument( + "--compatible-model-specifier", + help="the claim, as a PEP 508 requirement, e.g. 'policyengine-us==2.60.4'", + ) + parser.add_argument( + "--compatibility-claim-declared-by", + help="who declares the claim, recorded in the bundle", + ) + args = parser.parse_args(argv) + if args.certify and ( + args.output_dir is None + or args.baseline_scores is None + or args.tested_scores is None + or args.compatible_model_specifier is None + or args.compatibility_claim_declared_by is None + ): + parser.error( + "--certify needs --output-dir, --baseline-scores, --tested-scores, " + "--compatible-model-specifier and --compatibility-claim-declared-by" + ) + try: + if args.certify: + result = certify_recertification( + args.release_dir, + args.output_dir, + artifact_root=args.artifact_root, + compatibility_wheels=tuple(args.compatibility_wheel), + baseline_scores=args.baseline_scores, + tested_scores=args.tested_scores, + counterfactual_scores=args.counterfactual_scores, + compatible_model_specifier=args.compatible_model_specifier, + compatibility_claim_declared_by=args.compatibility_claim_declared_by, + ) + print( + json.dumps( + { + "certified_bundle": str(result), + "published": False, + "claim_measurement": recorded_claim_measurement(result), + } + ) + ) + else: + report = validate_recertification_candidate( + args.release_dir, + artifact_root=args.artifact_root, + require_compatibility=args.require_compatibility, + compatibility_wheels=tuple(args.compatibility_wheel), + ) + verdict = { + "valid": True, + "compatibility": report["compatibility"]["status"], + } + measurement = recorded_claim_measurement(args.release_dir) + if measurement: + verdict["claim_measurement"] = measurement + print(json.dumps(verdict)) + except (ValueError, OSError, ImportError, KeyError, TypeError) as exc: + print(str(exc), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 7af3891c3719804188c8bdfbd2b760541f4274b9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 12:04:44 -0400 Subject: [PATCH 02/12] Wire the recertification lane: builder tool, contract dispatch, publish-time parent check Co-Authored-By: Claude Opus 5.5 --- .../data/compatibility_recertification.py | 59 +- .../src/microcosm/data/contract.py | 25 + .../src/microcosm/data/release.py | 28 +- .../build_us_compatibility_recertification.py | 760 ++++++++++++++++++ 4 files changed, 867 insertions(+), 5 deletions(-) create mode 100644 tools/build_us_compatibility_recertification.py diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py index db847a917..2aeae1e05 100644 --- a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -772,7 +772,11 @@ class DriftMetric: ``variables`` are summed per unit of ``entity`` (a person variable over a tax-unit metric is first summed to the tax unit), except ``combine = "nonrefundable_ctc"``, which is ``max(min(ctc, ctc_limiting_tax_liability), - 0)`` as the release tool scores the SOI CTC rows. ``itemizers_only`` masks + 0)`` as the release tool scores the SOI CTC rows, and ``combine = + "part_b_irmaa"``, which is ``(gross_medicare_part_b_premium_if_enrolled - + base_part_b_premium) * medicare_enrolled``. ``entity = + "household_members"`` sums a person variable over each household, as the + release tool's ``policyengine_variable`` rows do. ``itemizers_only`` masks by ``tax_unit_itemizes``. The weight is the H5's household weight of the unit's household, so both engines weigh the same rows identically. """ @@ -876,6 +880,40 @@ class DriftMetric: STATISTIC_SUM, "usd", ), + # policyengine-us#10106 (in 2.60.4, not 2.2.1) lets IRMAA fall back to the + # earliest input year's income, so a single-year H5 gets payers; 2.60.4 + # also adds takes_up_medicare_part_d_if_eligible (default true). Part B + # IRMAA is the release's held-out definition (agent guide, microcosm#1206): + # gross premium if enrolled minus the base premium, over enrollees. + DriftMetric( + "medicare_part_b_irmaa_total", + "Medicare Part B income-related premium (IRMAA)", + "person", + ( + "gross_medicare_part_b_premium_if_enrolled", + "base_part_b_premium", + "medicare_enrolled", + ), + STATISTIC_SUM, + "usd", + combine="part_b_irmaa", + ), + DriftMetric( + "medicare_part_d_irmaa_total", + "Medicare Part D income-related surcharge", + "person", + ("income_adjusted_part_d_premium_surcharge",), + STATISTIC_SUM, + "usd", + ), + DriftMetric( + "medicare_cost_total", + "Medicare benefit cost", + "person", + ("medicare_cost",), + STATISTIC_SUM, + "usd", + ), DriftMetric( "adjusted_gross_income_total", "adjusted gross income", @@ -1296,6 +1334,8 @@ def combine_drift( "snap_total", "medicaid_total", "medicaid_enrollment", + "medicare_part_b_irmaa_total", + "medicare_part_d_irmaa_total", ) @@ -1433,6 +1473,21 @@ def covered(entries, name, version): version=model_version, declared_by=declared_by, ) + # The parent's own entries already cover what it was built with. A claim + # that also reaches back to that version spans every release between the + # two, none of which this recertification measured. + built_with = _mapping( + _mapping(parent_manifest.get("build")).get("built_with_model_package") + ).get("version") + if isinstance(built_with, str) and Version(built_with) in SpecifierSet( + model["specifier"] + ): + raise ValueError( + f"publisher compatibility claim {model['specifier']!r} reaches back to " + f"the built-with policyengine-us {built_with}; claim the measured " + f"version {model_version} (or a range above the built-with one), and " + "let the parent's entries carry the built-with version" + ) claims = {"model": model} if not covered(cores, "policyengine-core", core_version): claims["core"] = { @@ -1669,6 +1724,8 @@ def validate_recertification_candidate( failures.append( f"build.{field} must stay the parent's: the H5 was built with it" ) + if _mapping(manifest.get("build")).get("built_at") != parent_build.get("built_at"): + failures.append("build.built_at must stay the parent's: the H5 was built then") if _mapping(manifest.get("build")).get("recertified_from") != parent_id: failures.append("build.recertified_from must name the parent") for field in ( diff --git a/packages/microcosm-data/src/microcosm/data/contract.py b/packages/microcosm-data/src/microcosm/data/contract.py index 235dcca56..204aca25a 100644 --- a/packages/microcosm-data/src/microcosm/data/contract.py +++ b/packages/microcosm-data/src/microcosm/data/contract.py @@ -6756,6 +6756,31 @@ def validate_release_dir( compatibility_wheels=compatibility_wheels, ) return [] + from microcosm.data.compatibility_recertification import ( + RECERTIFICATION_RELEASE_TYPE, + validate_recertification_candidate, + ) + + if manifest_probe["release_type"] == RECERTIFICATION_RELEASE_TYPE: + # A published parent's bytes, re-measured under a newer + # engine: its own contract replays the parent's on the + # carried files. The child's H5 is the parent's, so it takes + # no parent_h5. + if parent_h5 is not None: + raise ReleaseContractError( + release_dir, + [ + "a compatibility recertification republishes its " + "parent's H5 unchanged and takes no parent_h5" + ], + ) + validate_recertification_candidate( + release_dir, + artifact_root=artifact_root, + require_compatibility=True, + compatibility_wheels=compatibility_wheels, + ) + return [] if manifest_probe["release_type"] != "calibration": raise ReleaseContractError( release_dir, diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index 9f91cc650..0cbd2b453 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -302,18 +302,28 @@ def prepare_release( """ release_dir = Path(release_dir) if parent_h5 is not None or compatibility_wheels: + from microcosm.data.compatibility_recertification import ( + RECERTIFICATION_RELEASE_TYPE, + ) from microcosm.data.source_enrichment import SOURCE_ENRICHMENT_RELEASE_TYPE manifest_path = release_dir / "release_manifest.json" manifest = ( json.loads(manifest_path.read_text()) if manifest_path.is_file() else {} ) - if ( - not isinstance(manifest, Mapping) - or manifest.get("release_type") != SOURCE_ENRICHMENT_RELEASE_TYPE + release_type = ( + manifest.get("release_type") if isinstance(manifest, Mapping) else None + ) + # A recertification replays its loader under the four wheels too, but + # its H5 is its parent's, so it takes no parent_h5. + if not ( + release_type == SOURCE_ENRICHMENT_RELEASE_TYPE + or (release_type == RECERTIFICATION_RELEASE_TYPE and parent_h5 is None) ): raise ValueError( - "parent_h5 and compatibility_wheels require a source_enrichment release." + "parent_h5 and compatibility_wheels require a source_enrichment " + "release; compatibility_wheels alone also serve a " + "compatibility_recertification release." ) contract_warnings: list[str] = [] if evidence: @@ -659,6 +669,16 @@ def publish_release( if api is None: api = _hf_api() + from microcosm.data.compatibility_recertification import ( + is_recertification, + require_parent_publication, + ) + + if is_recertification(prepared.release_dir): + # The offline gates proved the child against the parent manifest it + # carries; only the Hub can say that manifest is the published one. + # Checked before any branch, commit or upload. + require_parent_publication(api, prepared.release_dir, repo_id=repo_id) if evidence: payload = latest_evidence_pointer_payload( release_id, diff --git a/tools/build_us_compatibility_recertification.py b/tools/build_us_compatibility_recertification.py new file mode 100644 index 000000000..ee0d46641 --- /dev/null +++ b/tools/build_us_compatibility_recertification.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python3 +"""Build, and score, a compatibility recertification of a published US release. + +Three local steps; none publishes, stages or calibrates: + +- ``fetch-parent``: download a published parent's ``releases//`` files and + root artifacts at its immutable tag, and check every pinned digest. +- ``build``: package a pending ``compatibility_recertification`` candidate. The + parent's H5 and every other artifact are hard-linked (copied only across + devices) or copied byte for byte; nothing is derived. The contract runs + before the directory is exposed, atomically. +- ``score``: score the candidate's H5 under the installed policyengine-us, + in household batches, for every metric + ``microcosm.data.compatibility_recertification.drift_metric_definitions`` + defines. Run it once in an environment with the parent's built-with engine + and once (``--variant as_stored`` and ``--variant without_newly_read_inputs``) + in the tested environment; certification combines the three documents. + +Scoring runs baseline microsimulations over the whole population. Check +``reservation.py allowed`` first. See ``docs/us-compatibility-recertification.md``. +""" + +from __future__ import annotations + +import argparse +import contextlib +import errno +import gc +import json +import os +import platform +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from importlib import metadata +from pathlib import Path + +from microcosm.data import compatibility_recertification as recert +from microcosm.data import contract, source_enrichment, stored_inputs + +_PRODUCER_FILES = recert.PRODUCER_SOURCE_FILES +_ROOT = Path(__file__).resolve().parents[1] +_DEFAULT_BATCH_HOUSEHOLDS = 2_000 + + +def _producer_identity() -> dict: + """The clean commit and source digests publication authenticates.""" + + import h5py + + loaded = dict( + zip( + _PRODUCER_FILES, + ( + __file__, + recert.__file__, + source_enrichment.__file__, + stored_inputs.__file__, + contract.__file__, + ), + strict=True, + ) + ) + for filename, actual in loaded.items(): + if Path(actual).resolve() != (_ROOT / filename).resolve(): + raise ValueError( + f"Producer must execute this checkout's {filename}; install this " + "workspace's microcosm-data (editable) in the scoring environment" + ) + + def git(*args): + return subprocess.check_output(["git", *args], cwd=_ROOT, text=True).strip() + + return { + "repository": "https://github.com/PolicyEngine/microcosm", + "git_commit": git("rev-parse", "HEAD"), + "git_dirty": bool(git("status", "--porcelain")), + "runtime": { + "python": platform.python_version(), + "hdf5": h5py.version.hdf5_version, + **{ + package: metadata.version(package) + for package in ("numpy", "h5py", "microcosm-data") + }, + }, + "source_files_sha256": { + filename: recert.sha256_file(_ROOT / filename) + for filename in _PRODUCER_FILES + }, + } + + +# --------------------------------------------------------------------------- +# fetch-parent +# --------------------------------------------------------------------------- + + +def fetch_parent( + *, release_id: str, output_dir: Path, repo_id: str = recert.REPO_ID +) -> dict: + """Download a published parent at its tag into ``output_dir``; verify it.""" + + from huggingface_hub import HfApi, hf_hub_download + + if output_dir.exists(): + raise FileExistsError(f"{output_dir} exists; choose a new directory") + release_dir = output_dir / "releases" / release_id + artifact_root = output_dir / "artifacts" + release_dir.mkdir(parents=True) + artifact_root.mkdir() + + def download(path_in_repo: str, destination: Path) -> None: + local = hf_hub_download( + repo_id, + path_in_repo, + repo_type="dataset", + revision=release_id, + local_dir=destination, + ) + if not Path(local).is_file(): + raise FileNotFoundError(path_in_repo) + + prefix = f"releases/{release_id}/" + for name in ("release_manifest.json", "build_manifest.json"): + download(prefix + name, output_dir) + manifest = json.loads((release_dir / "release_manifest.json").read_text()) + # A release file is one the Hub serves under releases// at the tag; the + # rest are root artifacts, as the publisher uploaded them. + published = { + Path(item.path).name + for item in HfApi().list_repo_tree( + repo_id, repo_type="dataset", revision=release_id, path_in_repo=prefix + ) + } + for entry in manifest.get("artifacts", {}).values(): + path = entry["path"] + if Path(path).name == path and path in published: + download(prefix + path, output_dir) + else: + download(path, artifact_root) + return recert.describe_parent(release_dir, artifact_root) + + +# --------------------------------------------------------------------------- +# build +# --------------------------------------------------------------------------- + + +def _place(source: Path, destination: Path) -> None: + try: + os.link(source, destination) + except OSError as exc: + if exc.errno not in {errno.EXDEV, errno.EPERM, errno.ENOTSUP}: + raise + shutil.copyfile(source, destination) + destination.chmod(0o400) + + +def _expose_without_overwrite(staging: Path, output_dir: Path) -> None: + """Reserve ``output_dir`` with an exclusive mkdir, then rename onto it.""" + + occupied = "Output appeared during the build; refusing to overwrite" + try: + os.mkdir(output_dir, 0o700) + except FileExistsError: + raise FileExistsError(occupied) from None + try: + os.rename(staging, output_dir) + except OSError as exc: + with contextlib.suppress(OSError): + os.rmdir(output_dir) + if exc.errno in {errno.EEXIST, errno.ENOTEMPTY, errno.ENOTDIR}: + raise FileExistsError(occupied) from exc + raise + + +def build_candidate( + *, + parent_release_dir: Path, + parent_artifact_root: Path, + output_dir: Path, + release_id: str, +) -> dict: + """Package a pending recertification of the parent; return its report.""" + + parent = recert.describe_parent(parent_release_dir, parent_artifact_root) + if ( + not release_id.startswith(recert.US_RELEASE_PREFIX) + or Path(release_id).name != release_id + or release_id == parent["build_id"] + or "\\" in release_id + ): + raise ValueError("Choose a new bare US release id for the recertification") + if output_dir.exists() or output_dir.is_symlink(): + raise FileExistsError("Output directory already exists; choose a new candidate") + producer = _producer_identity() + parent_manifest = json.loads( + (parent_release_dir / "release_manifest.json").read_text() + ) + output_dir.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=".recertification-", dir=output_dir.parent)) + try: + os.chmod(staging, 0o700) + release_dir = staging / "releases" / release_id + release_dir.mkdir(parents=True, mode=0o700) + artifact_root = staging / "artifacts" + artifact_root.mkdir(mode=0o700) + for path in parent["root_artifacts"]: + destination = artifact_root / path + destination.parent.mkdir(parents=True, exist_ok=True) + _place(parent_artifact_root / path, destination) + for name in parent["release_files"]: + shutil.copyfile(parent_release_dir / name, release_dir / name) + shutil.copyfile( + parent_release_dir / "release_manifest.json", + release_dir / recert.PARENT_RELEASE_MANIFEST_FILE, + ) + shutil.copyfile( + parent_release_dir / "build_manifest.json", + release_dir / recert.PARENT_BUILD_MANIFEST_FILE, + ) + report = { + "schema_version": recert.REPORT_SCHEMA_VERSION, + "release_type": recert.RECERTIFICATION_RELEASE_TYPE, + "parent": parent, + "dataset": parent["dataset"], + "transformation": "none", + "parent_contract": recert.replay_parent_contract( + release_dir, parent, artifact_root=artifact_root + ), + "compatibility": {"status": "pending"}, + } + if report["parent_contract"]["other_failures"]: + raise ValueError( + "The parent fails the release contract on its own bytes: " + + " ".join(report["parent_contract"]["other_failures"]) + ) + recert.write_json(release_dir / recert.REPORT_FILE, report) + recert.write_json( + release_dir / "build_manifest.json", + { + "build_id": release_id, + "release_type": recert.RECERTIFICATION_RELEASE_TYPE, + "recertified_from": parent["build_id"], + "code": producer, + "dataset": parent["dataset"], + "calibration": { + "mode": "inherited", + "parent_build_id": parent["build_id"], + "diagnostics_sha256": parent["release_files"][ + "calibration_diagnostics.json" + ], + }, + "staging": { + "enabled": False, + "reason": "local recertification candidate; no calibration", + }, + }, + ) + parent_build = parent_manifest["build"] + manifest = { + key: parent_manifest[key] + for key in ( + "schema_version", + "dataset_role", + "default_datasets", + "calibration_diagnostics", + "compatible_model_packages", + "compatible_core_packages", + ) + if key in parent_manifest + } + manifest["release_type"] = recert.RECERTIFICATION_RELEASE_TYPE + manifest["data_package"] = { + "name": "microcosm-data", + "version": metadata.version("microcosm-data"), + } + manifest["build"] = { + "build_id": release_id, + "recertified_from": parent["build_id"], + **{ + key: parent_build[key] + for key in ( + "built_at", + "built_with_core_package", + "built_with_model_package", + ) + if key in parent_build + }, + } + artifacts = {} + for key, entry in parent_manifest["artifacts"].items(): + artifacts[key] = {**entry, "revision": release_id} + for key, name in recert.CHILD_ARTIFACT_KEYS.items(): + artifacts[key] = { + "kind": "evidence", + "path": name, + "repo_id": recert.REPO_ID, + "revision": release_id, + "sha256": recert.sha256_file(release_dir / name), + } + manifest["artifacts"] = artifacts + recert.write_json(release_dir / "release_manifest.json", manifest) + for path in release_dir.iterdir(): + path.chmod(0o400) + recert.validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + if ( + _producer_identity()["source_files_sha256"] + != producer["source_files_sha256"] + ): + raise ValueError( + "Producer source files changed while building the candidate" + ) + _expose_without_overwrite(staging, output_dir) + return report + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +# --------------------------------------------------------------------------- +# score +# --------------------------------------------------------------------------- + + +def _household_batches(household_ids, batch_households: int): + for start in range(0, len(household_ids), batch_households): + yield household_ids[start : start + batch_households] + + +def _state_batches(household_ids, state_fips): + for state in sorted(set(state_fips.tolist())): + yield household_ids[state_fips == state] + + +class _BatchFrames: + """The country loader's tables, sliced to whole households.""" + + def __init__(self, data, *, withheld: Sequence[str]): + self.tables = {} + for entity in recert._ENTITIES: + frame = getattr(data, entity) + drop = [column for column in withheld if column in frame.columns] + self.tables[entity] = frame.drop(columns=drop) if drop else frame + person = self.tables["person"] + for group in recert._ENTITIES[1:]: + column = f"person_{group}_id" + nested = person.groupby(column)["person_household_id"].nunique() + if (nested != 1).any(): + raise ValueError(f"{group} units must nest in one household") + + def batch(self, household_ids): + person = self.tables["person"] + mask = person["person_household_id"].isin(household_ids) + people = person.loc[mask] + tables = {"person": people.reset_index(drop=True)} + for group in recert._ENTITIES[1:]: + frame = self.tables[group] + ids = people[f"person_{group}_id"].unique() + tables[group] = frame.loc[frame[f"{group}_id"].isin(ids)].reset_index( + drop=True + ) + return tables + + +def _engine_known_periods(simulation, variables) -> set[tuple[str, str]]: + known = set() + for variable in variables: + if variable not in simulation.tax_benefit_system.variables: + continue + known.update( + (variable, str(period)) + for period in simulation.get_holder(variable).get_known_periods() + ) + return known + + +def _values(simulation, variable): + import numpy as np + + result = simulation.calculate(variable, recert.SCORING_PERIOD) + values = getattr(result, "values", result) + if hasattr(values, "decode_to_str"): + values = values.decode_to_str() + return np.asarray(values) + + +class _BatchScorer: + """Scores one batch engine for a set of metric definitions.""" + + def __init__(self, simulation, tables): + import numpy as np + + self.simulation = simulation + self.tables = tables + household = tables["household"] + weight = dict( + zip( + household["household_id"].to_numpy(), + household["household_weight"].to_numpy(), + strict=True, + ) + ) + person = tables["person"] + self.person_weight = np.asarray( + [weight[h] for h in person["person_household_id"].to_numpy()], + dtype=np.float64, + ) + self.group_weight = { + "household": household["household_weight"].to_numpy(np.float64) + } + self.person_group_position = {} + for group in recert._ENTITIES[1:]: + ids = tables[group][f"{group}_id"].to_numpy() + engine_ids = _values(simulation, f"{group}_id") + if not np.array_equal(engine_ids.astype(ids.dtype), ids): + raise ValueError(f"engine {group} order differs from the batch table") + position = {value: index for index, value in enumerate(ids)} + self.person_group_position[group] = np.asarray( + [position[value] for value in person[f"person_{group}_id"].to_numpy()] + ) + if group != "household": + first = np.full(len(ids), -1) + first[self.person_group_position[group][::-1]] = np.arange(len(person))[ + ::-1 + ] + self.group_weight[group] = self.person_weight[first] + engine_people = _values(simulation, "person_id") + if not np.array_equal( + engine_people.astype(person["person_id"].dtype), + person["person_id"].to_numpy(), + ): + raise ValueError("engine person order differs from the batch table") + self.entity_of = { + name: variable.entity.key + for name, variable in simulation.tax_benefit_system.variables.items() + } + + def _at(self, variable, entity): + import numpy as np + + values = _values(self.simulation, variable).astype(np.float64) + source = self.entity_of[variable] + target = "household" if entity == "household_members" else entity + if source == target: + return values + if source != "person": + raise ValueError(f"{variable} is a {source} variable, not one of {target}") + out = np.zeros(len(self.group_weight[target]), dtype=np.float64) + np.add.at(out, self.person_group_position[target], values) + return out + + def total(self, metric) -> float: + import numpy as np + + entity = metric["entity"] + arrays = [self._at(variable, entity) for variable in metric["variables"]] + if metric["combine"] == "nonrefundable_ctc": + ctc, limit = arrays + values = np.maximum(np.minimum(ctc, limit), 0.0) + elif metric["combine"] == "part_b_irmaa": + gross, base, enrolled = arrays + values = (gross - base) * enrolled + else: + values = np.sum(arrays, axis=0) + if metric["itemizers_only"]: + values = values * self._at("tax_unit_itemizes", "tax_unit") + statistic = metric["statistic"] + if statistic != recert.STATISTIC_SUM: + values = recert.signed_component(values, metric["source_variable"] or "") + if statistic == recert.STATISTIC_POSITIVE_COUNT: + values = (values > 0).astype(np.float64) + target = "household" if entity == "household_members" else entity + weights = ( + self.person_weight if target == "person" else self.group_weight[target] + ) + return float(np.dot(weights, values)) + + def filing_status(self, counts): + import numpy as np + + engine = _values(self.simulation, "filing_status").astype(str) + table = self.tables["tax_unit"] + stated = ( + table["filing_status_input"].astype(str).to_numpy() + if "filing_status_input" in table.columns + else np.full(len(engine), "UNSTATED") + ) + weights = self.group_weight["tax_unit"] + for s, e, w in zip(stated, engine, weights, strict=True): + cell = counts.setdefault(s, {}).setdefault( + e, {"tax_units": 0, "weighted": 0.0} + ) + cell["tax_units"] += 1 + cell["weighted"] += float(w) + + def roles(self, counts): + import numpy as np + + person = self.tables["person"] + if "tax_unit_role_input" not in person.columns: + return + head = _values(self.simulation, "is_tax_unit_head").astype(bool) + spouse = _values(self.simulation, "is_tax_unit_spouse").astype(bool) + engine = np.where(head, "HEAD", np.where(spouse, "SPOUSE", "DEPENDENT")) + stored = person["tax_unit_role_input"].astype(str).to_numpy() + differs = engine != stored + counts["persons"] += len(stored) + counts["persons_differing"] += int(differs.sum()) + counts["weighted_persons_differing"] += float(self.person_weight[differs].sum()) + units = np.unique(self.person_group_position["tax_unit"][differs]) + counts["tax_units_differing"] += len(units) + counts["weighted_tax_units_differing"] += float( + self.group_weight["tax_unit"][units].sum() + ) + + +def score_candidate( + *, + release_dir: Path, + artifact_root: Path, + output: Path, + variant: str, + batch_households: int = _DEFAULT_BATCH_HOUSEHOLDS, + microsimulation_cls=None, +) -> dict: + """Score the candidate's H5 under the installed engine; write the document.""" + + import numpy as np + from policyengine_us.data import USSingleYearDataset + + if microsimulation_cls is None: + from policyengine_us import Microsimulation + + microsimulation_cls = Microsimulation + if output.exists(): + raise FileExistsError(f"{output} exists") + report = json.loads((release_dir / recert.REPORT_FILE).read_text()) + dataset = report["dataset"] + h5 = artifact_root / dataset["filename"] + if recert.sha256_file(h5) != dataset["sha256"]: + raise ValueError("The candidate H5 differs from its report") + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + spm = ((diagnostics.get("build") or {}).get("post_export_scoring") or {}).get("spm") + if not isinstance(spm, dict): + raise ValueError( + "The parent's diagnostics record no post-export SPM selection to score under" + ) + definitions = recert.drift_metric_definitions(diagnostics) + tables = stored_inputs.h5_stored_tables(h5) + engine = stored_inputs.installed_us_engine() + changes = recert.stored_column_changes( + tables, + engine_variables=engine.variables, + stored_input_register=stored_inputs.US_STORED_NON_VARIABLE_COLUMNS, + ) + if variant == recert.VARIANT_AS_STORED: + withheld: list[str] = [] + elif variant == recert.VARIANT_WITHOUT_NEWLY_READ: + withheld = sorted(item["column"] for item in changes[recert.CHANGE_READ]) + if not withheld: + raise ValueError( + "This engine newly reads no stored column; nothing to withhold" + ) + else: + raise ValueError(f"unknown variant {variant!r}") + verdict = stored_inputs.h5_verdict(h5, engine=engine) + producer = _producer_identity() + frames = _BatchFrames(USSingleYearDataset(file_path=str(h5)), withheld=withheld) + household = frames.tables["household"] + household_ids = household["household_id"].to_numpy() + metrics = {metric["metric_id"]: metric for metric in definitions["metrics"]} + totals = dict.fromkeys(metrics, 0.0) + filing_status: dict = {} + roles = dict.fromkeys( + ( + "persons", + "persons_differing", + "weighted_persons_differing", + "tax_units_differing", + "weighted_tax_units_differing", + ), + 0, + ) + passes = [] + for batching, batches, allowed in ( + ( + recert.BATCHING_HOUSEHOLD, + _household_batches(household_ids, batch_households), + (), + ), + ( + recert.BATCHING_STATE, + _state_batches(household_ids, household["state_fips"].to_numpy()), + recert.STATE_AGGREGATE_VARIABLES, + ), + ): + selected = [m for m in metrics.values() if m["batching"] == batching] + count = 0 + for ids in batches: + batch = frames.batch(ids) + simulation = microsimulation_cls( + dataset=USSingleYearDataset(**batch, time_period=recert.SCORING_PERIOD), + spm=dict(spm), + ) + known = _engine_known_periods( + simulation, recert.POPULATION_AGGREGATE_VARIABLES + ) + scorer = _BatchScorer(simulation, batch) + for metric in selected: + totals[metric["metric_id"]] += scorer.total(metric) + if batching == recert.BATCHING_HOUSEHOLD: + scorer.filing_status(filing_status) + scorer.roles(roles) + computed = ( + _engine_known_periods(simulation, recert.POPULATION_AGGREGATE_VARIABLES) + - known + ) + refused = sorted(v for v, _ in computed if v not in allowed) + if refused: + raise RuntimeError( + f"A {batching} batch engine computed {refused}, which aggregate " + "over more than one batch can hold; this metric set cannot be " + "scored in batches (microcosm#956)." + ) + count += 1 + del simulation, scorer, batch + gc.collect() + passes.append( + {"batching": batching, "batches": count, "metrics": len(selected)} + ) + packages = source_enrichment._runtime_package_identities((), require_wheels=False) + document = { + "schema_version": recert.SCORES_SCHEMA_VERSION, + "kind": recert.SCORES_KIND, + "dataset": dict(dataset), + "variant": {"id": variant, "withheld_columns": withheld}, + "metric_definitions_sha256": recert.canonical_sha256(definitions), + "runtime": { + "python": platform.python_version(), + "packages": { + name: { + "version": identity["version"], + "source_sha256": identity["source_sha256"], + } + for name, identity in packages.items() + }, + }, + "engine": { + "label": engine.label, + "stored_input_verdict": { + key: verdict[key] + for key in ( + "passed", + "refused", + "registered_non_variables", + "register_sha256", + ) + }, + "column_changes": changes, + }, + "scoring": { + "method": "household_batched_country_loader", + "period": recert.SCORING_PERIOD, + "spm": spm, + "batch_households": batch_households, + "n_households": int(len(household_ids)), + "passes": passes, + "weights": "household_weight of each unit's household, from the H5", + }, + "metrics": {metric_id: {"value": value} for metric_id, value in totals.items()}, + "filing_status": filing_status, + "tax_unit_roles": roles, + "code": producer, + } + if not all(np.isfinite(value) for value in totals.values()): + raise ValueError("a scored total is not finite") + with output.open("x") as stream: + json.dump(document, stream, indent=2, sort_keys=True, allow_nan=False) + stream.write("\n") + return document + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + fetch = commands.add_parser("fetch-parent") + fetch.add_argument("--release-id", required=True) + fetch.add_argument("--output-dir", type=Path, required=True) + fetch.add_argument("--repo-id", default=recert.REPO_ID) + build = commands.add_parser("build") + build.add_argument("--parent-release-dir", type=Path, required=True) + build.add_argument("--parent-artifact-root", type=Path, required=True) + build.add_argument("--output-dir", type=Path, required=True) + build.add_argument("--release-id", required=True) + score = commands.add_parser("score") + score.add_argument("--release-dir", type=Path, required=True) + score.add_argument("--artifact-root", type=Path, required=True) + score.add_argument("--output", type=Path, required=True) + score.add_argument( + "--variant", + choices=(recert.VARIANT_AS_STORED, recert.VARIANT_WITHOUT_NEWLY_READ), + default=recert.VARIANT_AS_STORED, + ) + score.add_argument( + "--batch-households", type=int, default=_DEFAULT_BATCH_HOUSEHOLDS + ) + args = parser.parse_args(argv) + try: + if args.command == "fetch-parent": + result = fetch_parent( + release_id=args.release_id, + output_dir=args.output_dir, + repo_id=args.repo_id, + ) + print( + json.dumps({"parent": result["build_id"], "dataset": result["dataset"]}) + ) + elif args.command == "build": + report = build_candidate( + parent_release_dir=args.parent_release_dir, + parent_artifact_root=args.parent_artifact_root, + output_dir=args.output_dir, + release_id=args.release_id, + ) + print( + json.dumps( + { + "dataset": report["dataset"], + "parent_contract": report["parent_contract"]["status"], + "compatibility": report["compatibility"], + }, + indent=2, + ) + ) + else: + document = score_candidate( + release_dir=args.release_dir, + artifact_root=args.artifact_root, + output=args.output, + variant=args.variant, + batch_households=args.batch_households, + ) + print( + json.dumps( + {"engine": document["engine"]["label"], "variant": args.variant} + ) + ) + except (ValueError, OSError, RuntimeError) as exc: + print(f"Compatibility recertification refused: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4124001a2ee2013e3fa2aa6400f3bd90080a9021 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 12:17:13 -0400 Subject: [PATCH 03/12] Test the recertification lane: preservation, identity, claims, drift, publication Co-Authored-By: Claude Opus 5.5 --- .../data/compatibility_recertification.py | 4 +- .../test_compatibility_recertification.py | 746 ++++++++++++++++++ .../compatibility_recertification.py | 325 ++++++++ 3 files changed, 1074 insertions(+), 1 deletion(-) create mode 100644 packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py create mode 100644 test_support/microcosm_data/compatibility_recertification.py diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py index 2aeae1e05..5e3680537 100644 --- a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -1049,6 +1049,7 @@ def drift_metric_definitions(calibration_diagnostics: Mapping) -> dict: unscored. """ + from microcosm.data.contract import _target_registry_family from microcosm.data.us_critical_targets import ( US_EXACT_CRITICAL_TARGET_FIT_REQUIREMENTS, US_SOI_TABLE_1_4_NATIONAL_DOLLAR_FIT_REQUIREMENT, @@ -1065,7 +1066,8 @@ def drift_metric_definitions(calibration_diagnostics: Mapping) -> dict: metadata = _mapping(target.get("metadata")) if not name or name in seen or is_congressional_district_target(name, metadata): continue - family = name.split(".", 1)[0] + # The contract selects critical rows by registry family, so this does. + family = _target_registry_family(target) role = str(metadata.get("target_role") or "") requirement = next( ( diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py new file mode 100644 index 000000000..e1d126a0c --- /dev/null +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -0,0 +1,746 @@ +"""A recertification republishes a published parent unchanged and says what moved. + +Invariants (each held by tests below): + +- Byte preservation: every parent artifact reaches the child with the parent's + digest; changing one byte of any carried file refuses the child. +- Identity: the report's parent block is a function of the carried parent + manifest and files alone; editing either refuses the child. +- Additivity of compatibility: the child's compatibility lists are the + parent's entries followed by the claim, never fewer. +- Drift arithmetic: for every row, ``difference = tested - baseline`` and + ``newly_read_inputs_effect + other_engine_changes_effect = difference``. +- Classification: ``stored_column_changes`` partitions the model-named + columns it reports, and its ``ignored`` set is exactly what the stored-input + rule refuses. +- Publication: a child whose parent the Hub does not serve, byte for byte, + publishes nothing. +""" + +# ruff: noqa: F403, F405 + +from __future__ import annotations + +import json +import os +import shutil + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from microcosm.data import compatibility_recertification as recert +from microcosm.data import release as release_module +from microcosm.data.contract import ReleaseContractError, validate_release_dir +from microcosm.data.stored_inputs import undefined_stored_inputs +from test_support.microcosm_data.compatibility_recertification import * + + +def _rewrite_json(path, update): + os.chmod(path, 0o600) + payload = json.loads(path.read_text()) + update(payload) + path.write_text(json.dumps(payload)) + + +# --------------------------------------------------------------------------- +# The pending candidate. +# --------------------------------------------------------------------------- + + +def test_candidate_republishes_every_parent_artifact_byte_for_byte(parent, candidate): + parent_dir, parent_root = parent + release_dir, artifact_root = candidate + report = recert.validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + assert report["compatibility"] == {"status": "pending"} + assert report["transformation"] == "none" + assert report["parent"] == recert.describe_parent(parent_dir, parent_root) + for name in report["parent"]["release_files"]: + assert (release_dir / name).read_bytes() == (parent_dir / name).read_bytes() + for path in report["parent"]["root_artifacts"]: + assert (artifact_root / path).read_bytes() == (parent_root / path).read_bytes() + assert (release_dir / recert.PARENT_RELEASE_MANIFEST_FILE).read_bytes() == ( + parent_dir / "release_manifest.json" + ).read_bytes() + manifest = json.loads((release_dir / "release_manifest.json").read_text()) + parent_manifest = json.loads((parent_dir / "release_manifest.json").read_text()) + assert ( + manifest["compatible_model_packages"] + == parent_manifest["compatible_model_packages"] + ) + assert ( + manifest["build"]["built_with_model_package"] + == parent_manifest["build"]["built_with_model_package"] + ) + assert manifest["build"]["recertified_from"] == parent_dir.name + for key, entry in parent_manifest["artifacts"].items(): + assert manifest["artifacts"][key] == {**entry, "revision": CHILD_ID} + + +def test_pending_candidate_cannot_pass_the_publication_contract(candidate): + release_dir, artifact_root = candidate + with pytest.raises(ReleaseContractError, match="pending"): + validate_release_dir(release_dir, artifact_root=artifact_root) + + +def test_the_publication_contract_refuses_a_parent_h5(candidate): + release_dir, artifact_root = candidate + with pytest.raises(ReleaseContractError, match="takes no parent_h5"): + validate_release_dir( + release_dir, + artifact_root=artifact_root, + parent_h5=artifact_root / "populace_us_2024.h5", + ) + + +def _carried_files(candidate): + release_dir, artifact_root = candidate + report = json.loads((release_dir / recert.REPORT_FILE).read_text()) + return [ + *(release_dir / name for name in report["parent"]["release_files"]), + release_dir / recert.PARENT_RELEASE_MANIFEST_FILE, + release_dir / recert.PARENT_BUILD_MANIFEST_FILE, + *(artifact_root / path for path in report["parent"]["root_artifacts"]), + ] + + +@settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_changing_any_byte_of_any_carried_file_refuses( + candidate, tmp_path_factory, data +): + root = tmp_path_factory.mktemp("byte") + release_dir = root / "releases" / CHILD_ID + artifact_root = root / "artifacts" + shutil.copytree(candidate[0], release_dir) + shutil.copytree(candidate[1], artifact_root) + copy = (release_dir, artifact_root) + target = data.draw(st.sampled_from(_carried_files(copy))) + os.chmod(target, 0o600) + payload = bytearray(target.read_bytes()) + index = data.draw(st.integers(0, len(payload) - 1)) + payload[index] ^= data.draw(st.integers(1, 255)) + target.write_bytes(bytes(payload)) + with pytest.raises(ReleaseContractError): + recert.validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + + +@pytest.mark.parametrize( + "tamper, message", + [ + ( + lambda release_dir: _rewrite_json( + release_dir / recert.REPORT_FILE, + lambda report: report["parent"].update( + build_id="populace-us-2024-other" + ), + ), + "carried parent does not describe", + ), + ( + lambda release_dir: (release_dir / "extra.json").write_text("{}"), + "unexpected", + ), + ( + lambda release_dir: _rewrite_json( + release_dir / "release_manifest.json", + lambda manifest: manifest["compatible_model_packages"].append( + {"name": "policyengine-us", "specifier": "==2.60.4"} + ), + ), + "must carry the parent's compatible_model_packages", + ), + ( + lambda release_dir: _rewrite_json( + release_dir / "release_manifest.json", + lambda manifest: manifest["build"].update( + built_with_model_package={ + "name": "policyengine-us", + "version": "2.60.4", + } + ), + ), + "must stay the parent's", + ), + ( + lambda release_dir: _rewrite_json( + release_dir / recert.REPORT_FILE, + lambda report: report.update(transformation="reweighted"), + ), + "transformation", + ), + ], +) +def test_tampered_candidates_are_refused(candidate, tamper, message): + release_dir, artifact_root = candidate + tamper(release_dir) + with pytest.raises(ReleaseContractError, match=message): + recert.validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + + +def test_the_h5_may_not_live_in_the_release_directory(candidate): + release_dir, artifact_root = candidate + shutil.copyfile( + artifact_root / "populace_us_2024.h5", release_dir / "populace_us_2024.h5" + ) + with pytest.raises(ReleaseContractError, match="only in artifact_root"): + recert.validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + + +# --------------------------------------------------------------------------- +# Which parents a recertification accepts. +# --------------------------------------------------------------------------- + + +def test_build_refuses_reusing_the_parent_id(parent, tmp_path, monkeypatch): + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + release_dir, artifact_root = parent + with pytest.raises(ValueError, match="new bare US release id"): + tool.build_candidate( + parent_release_dir=release_dir, + parent_artifact_root=artifact_root, + output_dir=tmp_path / "out", + release_id=release_dir.name, + ) + assert not (tmp_path / "out").exists() + + +@pytest.mark.parametrize( + "update, message", + [ + ( + lambda m: m.update(release_type="source_enrichment"), + "not a calibration release", + ), + ( + lambda m: m.update(release_type=recert.RECERTIFICATION_RELEASE_TYPE), + "not a calibration", + ), + ( + lambda m: m.update(dataset_role="non_default_local_area"), + "not supported yet", + ), + ], +) +def test_describe_parent_refuses_unsupported_parents(parent, update, message): + release_dir, artifact_root = parent + _rewrite_json(release_dir / "release_manifest.json", update) + with pytest.raises(ValueError, match=message): + recert.describe_parent(release_dir, artifact_root) + + +def test_describe_parent_refuses_an_artifact_that_differs_from_its_pin(parent): + release_dir, artifact_root = parent + (artifact_root / "populace_us_2024.h5").write_bytes(b"other bytes") + with pytest.raises(ValueError, match="differs from its pin"): + recert.describe_parent(release_dir, artifact_root) + + +def test_parent_failing_its_own_contract_cannot_be_recertified( + parent, tmp_path, monkeypatch +): + release_dir, artifact_root = parent + _rewrite_json( + release_dir / "release_manifest.json", + lambda manifest: manifest.update(data_package={"name": "microcosm-data"}), + ) + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + with pytest.raises(ValueError, match="fails the release contract"): + tool.build_candidate( + parent_release_dir=release_dir, + parent_artifact_root=artifact_root, + output_dir=tmp_path / "out", + release_id=CHILD_ID, + ) + + +def _strip_stored_input_evidence(release_dir): + from microcosm.data.contract import US_STORED_INPUTS_EVIDENCE_KEY + + (release_dir / "stored_inputs.json").unlink() + for name in ("build_manifest.json",): + _rewrite_json( + release_dir / name, + lambda payload: ( + payload.pop(US_STORED_INPUTS_EVIDENCE_KEY, None), + payload.get("gate_evidence", {}).pop( + US_STORED_INPUTS_EVIDENCE_KEY, None + ), + ), + ) + + def strip_release(manifest): + manifest["build"].pop(US_STORED_INPUTS_EVIDENCE_KEY, None) + manifest["build"].get("gate_evidence", {}).pop( + US_STORED_INPUTS_EVIDENCE_KEY, None + ) + manifest["artifacts"].pop(US_STORED_INPUTS_EVIDENCE_KEY, None) + + _rewrite_json(release_dir / "release_manifest.json", strip_release) + + +def test_a_parent_without_stored_input_evidence_is_replaced_not_waived( + parent, tmp_path, monkeypatch +): + release_dir, artifact_root = parent + _strip_stored_input_evidence(release_dir) + with pytest.raises(ReleaseContractError, match="stored_inputs"): + validate_release_dir(release_dir, artifact_root=artifact_root) + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + tool.build_candidate( + parent_release_dir=release_dir, + parent_artifact_root=artifact_root, + output_dir=tmp_path / "candidate", + release_id=CHILD_ID, + ) + child = tmp_path / "candidate" / "releases" / CHILD_ID + report = json.loads((child / recert.REPORT_FILE).read_text()) + contract = report["parent_contract"] + assert contract["status"] == recert.PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED + assert contract["replaced_failures"] and not contract["other_failures"] + + # The replacement is the baseline engine's own verdict: a failing one refuses. + candidate = (child, tmp_path / "candidate" / "artifacts") + monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) + monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) + scores = write_scores(tmp_path / "scores", child) + baseline = json.loads(scores["baseline"].read_text()) + baseline["engine"]["stored_input_verdict"]["passed"] = False + scores["baseline"].write_text(json.dumps(baseline)) + with pytest.raises(ValueError, match="passing stored-input verdict"): + recert.certify_recertification( + candidate[0], + tmp_path / "certified" / CHILD_ID, + artifact_root=candidate[1], + compatibility_wheels=(), + baseline_scores=scores["baseline"], + tested_scores=scores["tested"], + counterfactual_scores=scores["counterfactual"], + compatible_model_specifier="policyengine-us==2.60.4", + compatibility_claim_declared_by=DECLARER, + ) + + +# --------------------------------------------------------------------------- +# The stored-column classification and its reviewed register. +# --------------------------------------------------------------------------- + +_COLUMN = st.sampled_from( + [ + "age", + "income_tax", + "first_home_mortgage_interest", + "tax_unit_role_input", + "person_source_id", + "A_AGE", + "parent_1_id", + "novel_column", + "_hidden", + ] +) + + +@settings(max_examples=200, deadline=None) +@given( + tables=st.dictionaries( + st.sampled_from(["person", "tax_unit"]), st.lists(_COLUMN), max_size=2 + ), + variables=st.sets(_COLUMN), + register=st.sets(_COLUMN), +) +def test_stored_column_changes_partition_and_match_the_stored_input_rule( + tables, variables, register +): + register = {column: "reason" for column in register} + changes = recert.stored_column_changes( + tables, engine_variables=variables, stored_input_register=register + ) + ignored = {item["column"] for item in changes[recert.CHANGE_IGNORED]} + read = {item["column"] for item in changes[recert.CHANGE_READ]} + stored = {column for columns in tables.values() for column in columns} + assert not ignored & read + assert ignored == set( + undefined_stored_inputs(stored, engine_variables=variables, register=register) + ) + assert read == { + c + for c in stored + if c in variables and c in register and c.islower() and not c.startswith("_") + } + + +def test_unreviewed_or_misdirected_changes_are_refused(): + changes = { + recert.CHANGE_IGNORED: [{"column": "novel_column", "tables": ["person"]}], + recert.CHANGE_READ: [ + {"column": "first_home_mortgage_interest", "tables": ["tax_unit"]} + ], + } + failures = recert.reviewed_column_change_failures(changes) + assert len(failures) == 2 + assert "no reviewed entry" in failures[0] + assert "says the engine change makes it 'ignored'" in failures[1] + + +def test_every_register_entry_has_a_direction_and_a_reason(): + for column, entry in recert.RECERTIFICATION_COLUMN_CHANGES.items(): + assert entry.change in {recert.CHANGE_IGNORED, recert.CHANGE_READ}, column + assert entry.reason.strip(), column + + +# --------------------------------------------------------------------------- +# The claim. +# --------------------------------------------------------------------------- + + +def _parent_manifest(candidate): + return json.loads((candidate[0] / recert.PARENT_RELEASE_MANIFEST_FILE).read_text()) + + +def test_claim_keeps_every_parent_entry_and_adds_the_measured_versions(candidate): + parent_manifest = _parent_manifest(candidate) + models, cores, claims = recert.claim_entries( + parent_manifest, + tested_packages=TESTED_PACKAGES, + specifier="==2.60.4", + declared_by=DECLARER, + ) + assert models[:-1] == parent_manifest["compatible_model_packages"] + assert ( + models[-1] + == claims["model"] + == { + "name": "policyengine-us", + "specifier": "==2.60.4", + "basis": "publisher_claim", + "declared_by": DECLARER, + } + ) + assert cores[:-1] == parent_manifest["compatible_core_packages"] + assert cores[-1]["specifier"] == "==3.33.3" + + +@pytest.mark.parametrize( + "specifier, message", + [ + (">=1.0,<2.61", "reaches back to the built-with"), + (">=2.60.4", "reaches 3.0.0"), + ("==2.60.3", "excludes the tested"), + ], +) +def test_claims_that_overreach_are_refused(candidate, specifier, message): + with pytest.raises(ValueError, match=message): + recert.claim_entries( + _parent_manifest(candidate), + tested_packages=TESTED_PACKAGES, + specifier=specifier, + declared_by=DECLARER, + ) + + +def test_a_parent_that_already_covers_the_tested_version_needs_no_child(candidate): + packages = {**TESTED_PACKAGES, "policyengine-us": {"version": "1.729.0"}} + with pytest.raises(ValueError, match="nothing to recertify"): + recert.claim_entries( + _parent_manifest(candidate), + tested_packages=packages, + specifier="==1.729.0", + declared_by=DECLARER, + ) + + +# --------------------------------------------------------------------------- +# Certification and drift. +# --------------------------------------------------------------------------- + + +def test_certified_child_passes_the_publication_contract( + candidate, tmp_path, monkeypatch +): + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + validate_release_dir(output, artifact_root=artifact_root) + report = json.loads((output / recert.REPORT_FILE).read_text()) + compatibility = report["compatibility"] + assert compatibility["publisher_claims"]["model"]["declared_by"] == DECLARER + measurement = compatibility["claim_measurement"]["model"] + assert measurement["measured_versions"] == ["2.60.4"] + assert measurement["baseline_version"] == "1.729.0" + summary = measurement["drift"]["summary"] + assert summary["exact_critical_rows"] == 18 + assert summary["headline"]["income_tax_net"][ + "relative_difference" + ] == pytest.approx(0.1) + read = compatibility["stored_column_changes"][recert.CHANGE_READ] + assert ( + read[0]["reason"] + == recert.RECERTIFICATION_COLUMN_CHANGES["tax_unit_role_input"].reason + ) + + +@pytest.mark.parametrize( + "path, update", + [ + ( + recert.DRIFT_FILE, + lambda drift: drift["rows"][0].update(tested=0.0), + ), + ( + recert.REPORT_FILE, + lambda report: report["compatibility"]["publisher_claims"]["model"].update( + specifier=">=2.60.4,<2.61" + ), + ), + ( + "release_manifest.json", + lambda manifest: manifest["compatible_model_packages"][-1].update( + specifier=">=2.60.4,<2.61" + ), + ), + ( + recert.REPORT_FILE, + lambda report: report["compatibility"]["claim_measurement"]["model"].update( + measured_versions=["2.60.4", "2.60.5"] + ), + ), + ], +) +def test_tampered_certified_children_are_refused( + candidate, tmp_path, monkeypatch, path, update +): + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + _rewrite_json(output / path, update) + with pytest.raises(ReleaseContractError): + validate_release_dir(output, artifact_root=artifact_root) + + +def test_drift_refuses_a_baseline_on_another_engine(candidate): + release_dir, _ = candidate + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + report = json.loads((release_dir / recert.REPORT_FILE).read_text()) + baseline = score_document(release_dir, packages=TESTED_PACKAGES) + with pytest.raises(ValueError, match="built-with policyengine-us"): + recert.combine_drift( + calibration_diagnostics=diagnostics, + dataset_sha256=report["dataset"]["sha256"], + baseline=baseline, + tested=score_document(release_dir), + counterfactual=None, + baseline_engine={ + "model": {"version": "1.729.0"}, + "core": {"version": "3.19.0"}, + }, + tested_packages=TESTED_PACKAGES, + newly_read_columns=[], + ) + + +def test_drift_requires_the_counterfactual_when_columns_are_newly_read(candidate): + release_dir, _ = candidate + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + report = json.loads((release_dir / recert.REPORT_FILE).read_text()) + with pytest.raises(ValueError, match="counterfactual"): + recert.combine_drift( + calibration_diagnostics=diagnostics, + dataset_sha256=report["dataset"]["sha256"], + baseline=score_document(release_dir, packages=BASELINE_PACKAGES), + tested=score_document(release_dir), + counterfactual=None, + baseline_engine={ + "model": {"version": "1.729.0"}, + "core": {"version": "3.19.0"}, + }, + tested_packages=TESTED_PACKAGES, + newly_read_columns=["tax_unit_role_input"], + ) + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + base=st.floats(-1e12, 1e12, allow_nan=False), + new=st.floats(-1e12, 1e12, allow_nan=False), + withheld=st.floats(-1e12, 1e12, allow_nan=False), +) +def test_drift_rows_decompose_exactly(parent, base, new, withheld): + release_dir, _ = parent + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + definitions = recert.drift_metric_definitions(diagnostics) + dataset = {"artifact": "x", "filename": "x.h5", "sha256": "c" * 64} + + def document( + packages, value, variant=recert.VARIANT_AS_STORED, withheld_columns=() + ): + return { + "schema_version": recert.SCORES_SCHEMA_VERSION, + "kind": recert.SCORES_KIND, + "dataset": dataset, + "variant": {"id": variant, "withheld_columns": list(withheld_columns)}, + "metric_definitions_sha256": recert.canonical_sha256(definitions), + "runtime": {"packages": packages}, + "metrics": { + m["metric_id"]: {"value": value} for m in definitions["metrics"] + }, + "code": dict(FIXTURE_PRODUCER), + } + + drift = recert.combine_drift( + calibration_diagnostics=diagnostics, + dataset_sha256=dataset["sha256"], + baseline=document(BASELINE_PACKAGES, base), + tested=document(TESTED_PACKAGES, new), + counterfactual=document( + TESTED_PACKAGES, + withheld, + recert.VARIANT_WITHOUT_NEWLY_READ, + ["parent_1_id"], + ), + baseline_engine={ + "model": {"version": "1.729.0"}, + "core": {"version": "3.19.0"}, + }, + tested_packages=TESTED_PACKAGES, + newly_read_columns=["parent_1_id"], + ) + for row in drift["rows"]: + assert row["difference"] == new - base + assert row["newly_read_inputs_effect"] + row["other_engine_changes_effect"] == ( + pytest.approx(row["difference"], abs=1e-3) + ) + assert recert.drift_summary(drift) == recert.drift_summary( + json.loads(json.dumps(drift)) + ) + + +def test_signed_component_scores_soi_rows_as_the_release_tool_does(): + import numpy as np + + values = np.array([-5.0, 0.0, 7.0]) + assert recert.signed_component(values, "income_tax").tolist() == [0.0, 0.0, 7.0] + assert recert.signed_component(values, "adjusted_gross_income").tolist() == [ + -5.0, + 0.0, + 7.0, + ] + assert recert.signed_component(values, "estate_losses").tolist() == [5.0, 0.0, 0.0] + assert recert.signed_component(values, "capital_gains_losses").tolist() == [ + 0.0, + 0.0, + 7.0, + ] + + +# --------------------------------------------------------------------------- +# Publication. +# --------------------------------------------------------------------------- + + +def _seed_parent(hub, parent_dir): + tag = parent_dir.name + hub._commits["commit-parent"] = { + f"releases/{tag}/release_manifest.json": ( + parent_dir / "release_manifest.json" + ).read_bytes(), + f"releases/{tag}/build_manifest.json": ( + parent_dir / "build_manifest.json" + ).read_bytes(), + } + hub._refs[tag] = "commit-parent" + + +def _publish(hub, output, artifact_root, monkeypatch): + monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) + monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) + return release_module.publish_release( + output, + "policyengine/populace-us", + api=hub, + artifact_root=artifact_root, + compatibility_wheels=(), + update_latest=False, + tag_only=True, + notify=False, + ) + + +def test_publication_refuses_a_parent_the_hub_does_not_serve( + hub, parent, candidate, tmp_path, monkeypatch +): + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + with pytest.raises(ValueError, match="is not published"): + _publish(hub, output, artifact_root, monkeypatch) + assert hub.events == [] + + +def test_publication_refuses_a_parent_whose_published_manifest_differs( + hub, parent, candidate, tmp_path, monkeypatch +): + parent_dir, _ = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + _seed_parent(hub, parent_dir) + hub._commits["commit-parent"][ + f"releases/{parent_dir.name}/release_manifest.json" + ] = b"{}" + with pytest.raises(ValueError, match="differs from the carried"): + _publish(hub, output, artifact_root, monkeypatch) + assert hub.events == [] + + +def test_publication_of_a_published_parents_child_is_tag_only( + hub, parent, candidate, tmp_path, monkeypatch +): + parent_dir, _ = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + _seed_parent(hub, parent_dir) + _publish(hub, output, artifact_root, monkeypatch) + uploads = dict(hub.uploads) + assert ( + uploads["populace_us_2024.h5"] + == (artifact_root / "populace_us_2024.h5").read_bytes() + ) + assert f"releases/{CHILD_ID}/{recert.DRIFT_FILE}" in uploads + assert "latest.json" not in uploads + assert [tag["tag"] for tag in hub.tags] == [CHILD_ID] + + +def test_prepare_release_takes_wheels_but_not_a_parent_h5( + candidate, tmp_path, monkeypatch +): + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) + monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) + with pytest.raises(ValueError, match="compatibility_wheels alone"): + release_module.prepare_release( + output, + artifact_root=artifact_root, + parent_h5=artifact_root / "populace_us_2024.h5", + update_latest=False, + tag_only=True, + ) + wheel = tmp_path / "policyengine_us-2.60.4-py3-none-any.whl" + wheel.write_bytes(b"") + prepared = release_module.prepare_release( + output, + artifact_root=artifact_root, + compatibility_wheels=(wheel,), + update_latest=False, + tag_only=True, + ) + assert "populace_us_2024.h5" in prepared.root_artifacts diff --git a/test_support/microcosm_data/compatibility_recertification.py b/test_support/microcosm_data/compatibility_recertification.py new file mode 100644 index 000000000..0ba639c12 --- /dev/null +++ b/test_support/microcosm_data/compatibility_recertification.py @@ -0,0 +1,325 @@ +"""Fixtures for the compatibility recertification lane. + +``parent`` is the publisher suite's synthetic US release (``release_dir``, +``artifact_root``) made a real published parent: its root artifacts carry the +digests its manifests pin, its critical diagnostics rows carry the release +tool's materializer metadata, and it passes today's release contract. +``candidate`` is the pending child the tool builds from it. Engine-free tests +replace the loader replay with :func:`fake_replay` and score documents with +:func:`score_document`; none of these values is a measurement. +""" + +# ruff: noqa: F401, F403, F405 + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from microcosm.data import compatibility_recertification as recert +from microcosm.data.contract import validate_release_dir +from test_support.microcosm_data.release import * +from test_support.microcosm_data.release import _bind_stored_input_evidence + +CHILD_ID = "populace-us-2024-9f1260b-recert-pe-us-2.60.4" +TESTED_PACKAGES = { + "policyengine-us": {"version": "2.60.4", "source_sha256": "1" * 64}, + "policyengine-core": {"version": "3.33.3", "source_sha256": "2" * 64}, + "policyengine": {"version": "6.10.1", "source_sha256": "3" * 64}, + "spm-calculator": {"version": "1.0.0", "source_sha256": "4" * 64}, +} +BASELINE_PACKAGES = { + "policyengine-us": {"version": "1.729.0", "source_sha256": "5" * 64}, + "policyengine-core": {"version": "3.19.0", "source_sha256": "6" * 64}, + "policyengine": {"version": "6.10.1", "source_sha256": "3" * 64}, + "spm-calculator": {"version": "1.0.0", "source_sha256": "4" * 64}, +} +DECLARER = "Max Ghenis (fixture declarer, not a real declaration)" + + +def _soi(source, base, mode, *, itemized=False): + metadata = { + "materializer": "irs_soi_slice", + "source_variable": source, + "variable": source, + ("base_variables" if "," in base else "base_variable"): base, + "measure_mode": mode, + "agi_lower_bound": "-inf", + "agi_upper_bound": "inf", + "filing_status": "All", + } + if itemized: + metadata["itemized_only"] = "true" + return metadata + + +_HT2 = "irs_soi.ty2022.historic_table_2.us.all." +_EITC = "irs_soi.ty2024.filing_season_week47.eitc_all_returns.earned_income_credit." +_T21 = "irs_soi.ty2023.table_2_1.itemized_all_returns.all." +CRITICAL_METADATA = { + f"{_HT2}income_tax_liability_amount@2024": _soi("income_tax", "income_tax", "sum"), + f"{_HT2}income_tax_liability_returns@2024": _soi( + "income_tax", "income_tax", "indicator_sum" + ), + "ssa_supplement.cy2024.oasdi_ssi_payments.social_security_benefits." + "payment_amount@2024": { + "materializer": "policyengine_variable", + "base_variable": "social_security", + "measure_mode": "sum", + }, + f"{_HT2}ctc_amount@2024": _soi("ctc", "ctc,ctc_limiting_tax_liability", "sum"), + f"{_HT2}ctc_claims@2024": _soi( + "ctc", "ctc,ctc_limiting_tax_liability", "indicator_sum" + ), + f"{_HT2}actc_amount@2024": _soi("refundable_ctc", "refundable_ctc", "sum"), + f"{_HT2}actc_claims@2024": _soi( + "refundable_ctc", "refundable_ctc", "indicator_sum" + ), + f"{_EITC}total_earned_income_credit_amount@2024": _soi("eitc", "eitc", "sum"), + f"{_EITC}total_earned_income_credit_returns@2024": _soi( + "eitc", "eitc", "indicator_sum" + ), + f"{_HT2}premium_tax_credit_amount@2024": _soi( + "assigned_aca_ptc", "assigned_aca_ptc", "sum" + ), + f"{_HT2}premium_tax_credit_returns@2024": _soi( + "assigned_aca_ptc", "assigned_aca_ptc", "indicator_sum" + ), + f"{_HT2}taxable_social_security_amount@2024": _soi( + "taxable_social_security", "tax_unit_taxable_social_security", "sum" + ), + f"{_HT2}taxable_social_security_returns@2024": _soi( + "taxable_social_security", "tax_unit_taxable_social_security", "indicator_sum" + ), + f"{_T21}home_mortgage_interest_returns@2024": _soi( + "deductible_mortgage_interest", + "deductible_mortgage_interest_tax_unit", + "indicator_sum", + itemized=True, + ), + f"{_T21}home_mortgage_interest_amount@2024": _soi( + "deductible_mortgage_interest", + "deductible_mortgage_interest_tax_unit", + "sum", + itemized=True, + ), + f"{_HT2}itemized_deductions_amount@2024": _soi( + "itemized_taxable_income_deductions", + "itemized_taxable_income_deductions", + "sum", + itemized=True, + ), + f"{_HT2}limited_state_local_taxes_amount@2024": _soi( + "salt_deduction", "salt_deduction", "sum", itemized=True + ), + f"{_HT2}medical_dental_expense_amount@2024": _soi( + "medical_expense_deduction", "medical_expense_deduction", "sum", itemized=True + ), + "irs_soi.ty2023.table_1_4.all.wages_salaries_amount@2024": _soi( + "employment_income", "employment_income", "sum" + ), +} + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _rewrite(path: Path, update) -> None: + payload = json.loads(path.read_text()) + update(payload) + path.write_text(json.dumps(payload)) + + +def make_published_parent(release_dir: Path, artifact_root: Path) -> None: + """Pin the fixture's real artifact bytes and materializer metadata.""" + + h5 = artifact_root / "populace_us_2024.h5" + npz = artifact_root / "populace_us_2024_calibration.npz" + h5.write_bytes(b"synthetic parent microdata, not an HDF5 file") + npz.write_bytes(b"synthetic parent calibration weights") + diagnostics = release_dir / "calibration_diagnostics.json" + + def annotate(payload): + for target in payload["targets"]: + if target["name"] in CRITICAL_METADATA: + target["metadata"] = { + **target["metadata"], + **CRITICAL_METADATA[target["name"]], + } + payload["build"] = { + **payload.get("build", {}), + "post_export_scoring": {"spm": {"geography_kind": "county"}}, + } + + _rewrite(diagnostics, annotate) + + def pin_release(payload): + artifacts = payload["artifacts"] + artifacts["populace_us_2024"]["sha256"] = _sha(h5) + artifacts["populace_us_2024_calibration"]["sha256"] = _sha(npz) + artifacts["calibration_diagnostics"]["sha256"] = _sha(diagnostics) + payload["build"]["built_at"] = "2026-09-30T15:04:01+00:00" + + def pin_build(payload): + payload["dataset"]["sha256"] = _sha(h5) + payload["calibration"]["sha256"] = _sha(npz) + + _rewrite(release_dir / "release_manifest.json", pin_release) + _rewrite(release_dir / "build_manifest.json", pin_build) + _bind_stored_input_evidence(release_dir) + validate_release_dir(release_dir, artifact_root=artifact_root) + + +@pytest.fixture +def parent(release_dir: Path, artifact_root: Path) -> tuple[Path, Path]: + make_published_parent(release_dir, artifact_root) + return release_dir, artifact_root + + +FIXTURE_PRODUCER = { + "repository": "https://github.com/PolicyEngine/microcosm", + "git_commit": "a" * 40, + "git_dirty": False, + "runtime": {"python": "3.13.0"}, + "source_files_sha256": dict.fromkeys(recert.PRODUCER_SOURCE_FILES, "b" * 64), +} + + +def load_tool(): + import tools.build_us_compatibility_recertification as tool + + return tool + + +@pytest.fixture +def candidate(parent, tmp_path, monkeypatch) -> tuple[Path, Path]: + """The pending child the tool builds; returns (release_dir, artifact_root).""" + + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + release_dir, artifact_root = parent + output = tmp_path / "candidate" + tool.build_candidate( + parent_release_dir=release_dir, + parent_artifact_root=artifact_root, + output_dir=output, + release_id=CHILD_ID, + ) + return output / "releases" / CHILD_ID, output / "artifacts" + + +def fake_replay(changes=None, packages=None): + """A loader replay stand-in returning a fixed receipt (not a measurement).""" + + changes = changes or { + recert.CHANGE_IGNORED: [ + {"column": "first_home_mortgage_interest", "tables": ["tax_unit"]} + ], + recert.CHANGE_READ: [{"column": "tax_unit_role_input", "tables": ["person"]}], + } + receipt = { + "schema_version": recert.RECEIPT_SCHEMA_VERSION, + "status": "passed", + "packages": packages or TESTED_PACKAGES, + "stored_inputs": {"column_changes": changes}, + } + + def replay(candidate_h5, *, require_wheels=False, compatibility_wheels=()): + return json.loads(json.dumps(receipt)) + + return replay + + +def score_document( + release_dir: Path, + *, + packages=TESTED_PACKAGES, + variant=recert.VARIANT_AS_STORED, + withheld=(), + value=lambda metric_id: 1_000.0, + stored_input_passed=True, +) -> dict: + """A score document shaped as the tool writes it, with invented values.""" + + from microcosm.data.stored_inputs import register_sha256 + + report = json.loads((release_dir / recert.REPORT_FILE).read_text()) + diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) + definitions = recert.drift_metric_definitions(diagnostics) + return { + "schema_version": recert.SCORES_SCHEMA_VERSION, + "kind": recert.SCORES_KIND, + "dataset": report["dataset"], + "variant": {"id": variant, "withheld_columns": sorted(withheld)}, + "metric_definitions_sha256": recert.canonical_sha256(definitions), + "runtime": {"packages": packages}, + "engine": { + "label": f"policyengine-us {packages['policyengine-us']['version']}", + "stored_input_verdict": { + "passed": stored_input_passed, + "refused": [], + "registered_non_variables": [], + "register_sha256": register_sha256(), + }, + }, + "metrics": { + metric["metric_id"]: {"value": value(metric["metric_id"])} + for metric in definitions["metrics"] + }, + "filing_status": {}, + "tax_unit_roles": {}, + "code": dict(FIXTURE_PRODUCER), + } + + +def write_scores(directory: Path, release_dir: Path, **overrides) -> dict[str, Path]: + """Baseline, tested and counterfactual documents on disk, by role.""" + + directory.mkdir(parents=True, exist_ok=True) + documents = { + "baseline": score_document( + release_dir, + packages=BASELINE_PACKAGES, + value=overrides.get("baseline", lambda metric_id: 1_000.0), + ), + "tested": score_document( + release_dir, value=overrides.get("tested", lambda metric_id: 1_100.0) + ), + "counterfactual": score_document( + release_dir, + variant=recert.VARIANT_WITHOUT_NEWLY_READ, + withheld=("tax_unit_role_input",), + value=overrides.get("counterfactual", lambda metric_id: 1_040.0), + ), + } + paths = {} + for role, document in documents.items(): + paths[role] = directory / f"{role}.json" + paths[role].write_text(json.dumps(document)) + return paths + + +def certify(candidate, tmp_path, monkeypatch, *, specifier="policyengine-us==2.60.4"): + """Certify the candidate with the fake replay and fixture scores.""" + + release_dir, artifact_root = candidate + monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) + monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) + scores = write_scores(tmp_path / "scores", release_dir) + output = tmp_path / "certified" / release_dir.name + recert.certify_recertification( + release_dir, + output, + artifact_root=artifact_root, + compatibility_wheels=(), + baseline_scores=scores["baseline"], + tested_scores=scores["tested"], + counterfactual_scores=scores["counterfactual"], + compatible_model_specifier=specifier, + compatibility_claim_declared_by=DECLARER, + ) + return output, artifact_root From f393805fe3e1a2ee6924b7b80a77f173e56ba715 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 12:20:00 -0400 Subject: [PATCH 04/12] Document the recertification lane and pin its copies of release-tool rules Co-Authored-By: Claude Opus 5.5 --- README.md | 8 + .../us-compatibility-recertification.added.md | 1 + docs/agent-guide.md | 12 + docs/us-compatibility-recertification.md | 227 ++++++++++++++++++ ...us_compatibility_recertification_parity.py | 67 ++++++ 5 files changed, 315 insertions(+) create mode 100644 changelog.d/us-compatibility-recertification.added.md create mode 100644 docs/us-compatibility-recertification.md create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_compatibility_recertification_parity.py diff --git a/README.md b/README.md index e38a0d0e3..cb42f8fb3 100644 --- a/README.md +++ b/README.md @@ -348,6 +348,14 @@ type publishes the [reported-receipt child of the national default](docs/us-repo as a tag-only donor for the ACS local chain. It is never the `latest.json` default. +A [compatibility recertification](docs/us-compatibility-recertification.md) +republishes a published US calibration release byte for byte under a new id, +with a measured `compatible_model_packages` claim for a newer policyengine-us: +a loader replay under the four tested wheels, the engine's reviewed column +changes, and the drift of the critical rows and headline totals between the +built-with and tested engines. It never recalibrates. Publication refuses a +child whose parent tag on the Hub does not serve the carried manifest bytes. + The non-default ACS local-area chain (`tools/build_us_acs_local_release.py`) calibrates to the SOI `state` surface by default, the 4,459-target contract of Build O and Build P; `--soi-mode totals` and `--soi-mode full` are explicit diff --git a/changelog.d/us-compatibility-recertification.added.md b/changelog.d/us-compatibility-recertification.added.md new file mode 100644 index 000000000..f79c2855a --- /dev/null +++ b/changelog.d/us-compatibility-recertification.added.md @@ -0,0 +1 @@ +Add the `compatibility_recertification` release type: a published US calibration release republished byte for byte with a measured `compatible_model_packages` claim for a newer policyengine-us. It carries the parent's published manifests as its identity, replays the native loaders under the four tested wheels, records the engine's reviewed column changes in both directions, and embeds the drift of the critical rows and headline totals between the built-with and tested engines. Publication refuses a child whose parent tag does not serve the carried manifest bytes. `tools/build_us_compatibility_recertification.py` fetches the parent, builds the candidate and scores it; see `docs/us-compatibility-recertification.md`. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 10bc99fdd..83a859fff 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -305,6 +305,18 @@ against both H5 files. The publisher refuses to point `latest.json` at this child, so publish it with `--no-latest --tag-only`. See [the reported-receipt runbook](us-reported-receipt-source-enrichment.md). +`compatibility_recertification` is a separate release type +(`microcosm.data.compatibility_recertification`, +`tools/build_us_compatibility_recertification.py`): a published US +calibration release, republished byte for byte with a measured claim for a +newer policyengine-us. Its parent identity is the parent's own published +`release_manifest.json`, carried and hashed, never a code constant; the +publisher checks the parent tag on the Hub before any upload. A change in +what the tested engine reads or ignores must have an entry in +`RECERTIFICATION_COLUMN_CHANGES`. It takes `--compatibility-wheel` and no +`--parent-h5`. National parents only for now. See +[the recertification runbook](us-compatibility-recertification.md). + Three US release seams refuse a stored column that is lowercase snake_case, not a variable of the engine the release is certified against, and not in the reviewed register `microcosm.data.stored_inputs.US_STORED_NON_VARIABLE_COLUMNS` diff --git a/docs/us-compatibility-recertification.md b/docs/us-compatibility-recertification.md new file mode 100644 index 000000000..f465c8d0f --- /dev/null +++ b/docs/us-compatibility-recertification.md @@ -0,0 +1,227 @@ +# US compatibility recertification + +A `compatibility_recertification` release lets a published, certified US +calibration release carry a measured `compatible_model_packages` claim for a +newer policyengine-us without a rebuild. The child republishes the parent's +H5, calibration weights and every release file byte for byte under a new id +and tag. It transforms no data and never recalibrates. What it adds is +evidence about the newer engine: + +- **a loader replay** under the four tested wheels (country, Core, wrapper, + calculator); +- **the engine's column changes**: what the newer engine ignores that the + built-with one read, and what it reads that no engine read before; +- **the drift**: the 18 exact critical rows, the scorable national SOI Table + 1.4 dollar rows and the headline totals, scored on the same H5 under both + engines; +- **a claim**, declared by a named person after reading the drift. + +The module is `microcosm.data.compatibility_recertification`; the tool is +`tools/build_us_compatibility_recertification.py`. + +## Parent identity + +The parent is never a code constant. The child carries the parent's published +`release_manifest.json` and `build_manifest.json` byte for byte +(`parent_release_manifest.json`, `parent_build_manifest.json`) and records +their SHA-256 in `compatibility_recertification.json`. That manifest pins +every parent artifact, so it is the whole lineage. Offline, the validator: + +- rebuilds the parent from the carried files and requires the report's + `parent` block to equal what they describe; +- requires every parent artifact in the child with the parent's digest, at + the child's revision; +- replays today's release contract on the rebuilt parent (`parent_contract`). + +Publication adds the online anchor: after the Hub client exists and before +any branch, commit or upload, `publish_release` downloads both manifests at +the parent's tag and refuses unless they equal the carried bytes +(`require_parent_publication`). Preflight stays offline. + +The parent must be a US calibration release (`release_type` absent or +`calibration`) with `dataset_role` `national_default`. A recertification of a +recertification or of a source enrichment is refused, so every child is one +hop from the build its calibration evidence describes. Recertify the +calibration release again for a later engine. + +### Parents that predate microcosm#1034 + +A parent published before the stored-input evidence contract carries no +`stored_inputs.json`, so today's contract refuses it for that reason alone. +The recertification measures the same rule itself, against the current +register, under both engines: the baseline score records the built-with +engine's verdict and the loader replay grades the tested engine. Such a parent +passes as `passed_with_stored_input_evidence_replaced`, the replaced failure +lines are recorded verbatim, and certification requires the baseline verdict +to pass. Any other contract failure refuses the child. + +## What the newer engine reads differently + +`stored_column_changes` classifies every stored model-named column against +the tested engine and the stored-input register: + +- `ignored`: not a variable of the tested engine and not a register entry. + The stored-input rule refuses exactly these. The parent passed that rule + under its built-with engine, so that engine read them. +- `read`: a register entry (a column the register says no engine reads) that + the tested engine defines, so it now reads the column as an input. No + existing gate sees this direction. + +Each change needs an entry in `RECERTIFICATION_COLUMN_CHANGES` with its +direction and reason, or the replay refuses. The report records the changes +with their reasons. Measured against policyengine-us 2.60.4 on the Route A +release `populace-us-2024-0fb05b6-4b57d15a287c-20260930T150401Z`: + +| Column | Change | Why | +| --- | --- | --- | +| `first_home_mortgage_interest`, `second_home_mortgage_interest` | ignored | removed by policyengine-us#9605; interest comes from the stored person-level `home_mortgage_interest` | +| `tax_unit_role_input` | read | policyengine-us#9617: heads and spouses follow the stored roles, not age | +| `parent_1_id`, `parent_2_id` | read | policyengine-us#9406: Medicaid MAGI household composition | + +`filing_status_input` is a register entry that both engines ignore; both +derive `filing_status`. The drift records the stated-by-derived filing-status +table under each engine. + +## Drift + +The tool scores the H5 in household batches of the country loader's tables: +2,000 households by default, every group unit nested in one household. Each +engine is built with the SPM selection the parent's post-export scoring +recorded. The weight of every unit is its household's `household_weight` +from the H5, so both engines weigh the same rows identically. The rules: + +- **Critical rows** use the release tool's SOI semantics: the signed + component (positive part, except the identity and loss sources), the + nonrefundable-CTC rule and the itemizer mask. +- **Population aggregates are refused.** A batch engine that computed one of + the release tool's population-aggregate formulas refuses + (`POPULATION_AGGREGATE_VARIABLES`). +- **Medicaid is scored in batches of whole states.** Its two state sums are + `sum_by_state`, so whole-state batches compute them exactly. + +Three score runs feed one drift document: + +1. `--variant as_stored` under the built-with engine (the baseline); +2. `--variant as_stored` under the tested engine; +3. `--variant without_newly_read_inputs` under the tested engine, with the + `read` columns withheld. It is required whenever there are any, and it + splits each row's move into `newly_read_inputs_effect` (for example, the + role-driven move in income tax, EITC and CTC) and + `other_engine_changes_effect`. + +The drift document embeds all three, and validation recomputes every row from +them. The tested runs must be the exact runtime the loader replay certified +(version and installed-source digest of all four packages). The baseline run +must be the parent's built-with policyengine-us and policyengine-core. + +## Steps + +Every step up to preflight is local. Scoring runs population microsims: +check `python3 ~/reviews/fleet-ops/tools/reservation.py allowed ` +first, and stay out of reserved windows. Run everything from one clean, +reviewed checkout. Publication authenticates the producer files against the +recorded commit. + +1. **Fetch the parent** at its tag: + + ```bash + uv run --no-sync python tools/build_us_compatibility_recertification.py fetch-parent \ + --release-id PARENT_ID --output-dir /path/parent + ``` + +2. **Build the candidate.** Root artifacts are hard-linked; compatibility is + `pending`: + + ```bash + uv run --no-sync python tools/build_us_compatibility_recertification.py build \ + --parent-release-dir /path/parent/releases/PARENT_ID \ + --parent-artifact-root /path/parent/artifacts \ + --output-dir /path/candidate --release-id CHILD_ID + ``` + +3. **Score** in two isolated environments, each with this checkout's + `microcosm-data` installed editable. One has the parent's built-with + wheels, the other the four tested wheels: + + ```bash + BASE/bin/python tools/build_us_compatibility_recertification.py score \ + --release-dir /path/candidate/releases/CHILD_ID \ + --artifact-root /path/candidate/artifacts --output baseline.json + TESTED/bin/python tools/build_us_compatibility_recertification.py score ... --output tested.json + TESTED/bin/python tools/build_us_compatibility_recertification.py score ... \ + --variant without_newly_read_inputs --output counterfactual.json + ``` + +4. **Put the drift in front of the declarer.** Whether drift blocks the + claim is the declarer's call, not this lane's. Queue it with + `cos add-decision`. + +5. **Certify** in the tested environment with the declarer's ruling: + + ```bash + TESTED/bin/python -m microcosm.data.compatibility_recertification --certify \ + --release-dir /path/candidate/releases/CHILD_ID \ + --artifact-root /path/candidate/artifacts \ + --output-dir /path/certified/releases/CHILD_ID \ + --compatibility-wheel policyengine_us-X.whl --compatibility-wheel policyengine_core-Y.whl \ + --compatibility-wheel policyengine-Z.whl --compatibility-wheel spm_calculator-W.whl \ + --baseline-scores baseline.json --tested-scores tested.json \ + --counterfactual-scores counterfactual.json \ + --compatible-model-specifier 'policyengine-us==X' \ + --compatibility-claim-declared-by 'NAME (cos dNNNN)' + ``` + + The certified manifest keeps every parent entry. It adds one + `publisher_claim` model entry, which must contain the tested version, pass + the source-enrichment claim guards and not reach back to the built-with + version. It also adds the tested Core as an exact `publisher_claim` pin, + unless a parent entry already covers that Core. `build.built_with_*` + stays the parent's. + +6. **Preflight**, offline, and then **publish** on the owner's go: + + ```bash + microcosm-publish-release /path/certified/releases/CHILD_ID \ + --repo-id policyengine/populace-us \ + --artifact-root /path/candidate/artifacts \ + --compatibility-wheel ... (four) \ + --no-latest --tag-only --preflight-only + ``` + + Publication is the same command through `tools/publish_release.sh` without + `--preflight-only`. Whether the child should also become `latest.json` is + a separate decision; the commands above leave the pointer alone. + +7. **Certify the bundle downstream.** policyengine.py certifies the child on + its publisher claim (`compatibility_basis: legacy_compatible_model_package`, + with a warning naming the built-with version). + +## Limits + +- **The regional release.** policyengine.py's US certification also checks + the regional (ACS local-area) release against the pinned model. On + 2026-10-23 the routed regional release + `populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z` was built with + policyengine-us 2.2.1 and claimed nothing wider. Certifying 2.60.4 then + needs either `--drop-regional-release` (state and district simulations fall + back to the national file), a local-area recertification, or a fresh + local-area build on 2.60.x. Local-area parents are a follow-up: the identity + mechanism does not depend on the role, but the child must replay the + local-area contract. +- **One measured version.** The claim's `claim_measurement` names the one + version the loader replay and the drift measured. A range admits versions + nothing measured. +- **Not numerical acceptance.** The replay proves loading; the drift reports + movement; neither is a calibration to the new engine. + +## Local checks + +```bash +uv run --no-sync pytest \ + packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py \ + packages/microcosm-build/tests/engine_free/us/test_us_compatibility_recertification_parity.py +python3 tools/ci_test_plan.py verify +``` + +These tests use invented releases and score documents. Passing them does not +certify a real artifact. diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_compatibility_recertification_parity.py b/packages/microcosm-build/tests/engine_free/us/test_us_compatibility_recertification_parity.py new file mode 100644 index 000000000..b66388b3d --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_compatibility_recertification_parity.py @@ -0,0 +1,67 @@ +"""The recertification scorer copies two release-tool rules; hold them equal. + +``microcosm.data.compatibility_recertification`` scores SOI rows with the +release tool's signed-component rule and refuses the release tool's +population-aggregate formulas in batch engines. The data shard cannot import +the tool, so it carries copies, and these tests are what keeps them the +tool's. +""" + +from __future__ import annotations + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from microcosm.data import compatibility_recertification as recert +from test_support.microcosm_build.us_post_export_scoring import _load_builder_module + + +@pytest.fixture(scope="module") +def release_tool(): + return _load_builder_module() + + +def test_population_aggregate_variables_are_the_release_tools(release_tool): + assert recert.POPULATION_AGGREGATE_VARIABLES == ( + release_tool.US_POPULATION_AGGREGATE_VARIABLES + ) + assert set(recert.STATE_AGGREGATE_VARIABLES) <= set( + release_tool.US_POPULATION_AGGREGATE_VARIABLES + ) + + +_SOURCES = sorted( + { + "adjusted_gross_income", + "rent_and_royalty_net_income", + "capital_gains_losses", + "business_net_losses", + "estate_losses", + "partnership_and_s_corp_losses", + "rent_and_royalty_net_losses", + "miscellaneous_losses", + "income_tax", + "eitc", + "ctc", + "employment_income", + "taxable_social_security", + } +) + + +@settings(max_examples=300, deadline=None) +@given( + values=st.lists( + st.floats(-1e12, 1e12, allow_nan=False, allow_infinity=False), + min_size=1, + max_size=16, + ), + source=st.sampled_from(_SOURCES), +) +def test_signed_component_is_the_release_tools(release_tool, values, source): + np.testing.assert_array_equal( + recert.signed_component(values, source), + release_tool._signed_component(np.asarray(values), source), + ) From 4a27a634101604837fbac827e37b784758ecdb97 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 13:57:39 -0400 Subject: [PATCH 05/12] Score a written H5 under the real engine: batch-size invariance and the counterfactual Co-Authored-By: Claude Opus 5.5 --- ...t_compatibility_recertification_scoring.py | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py diff --git a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py new file mode 100644 index 000000000..1e3de32e5 --- /dev/null +++ b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py @@ -0,0 +1,191 @@ +"""The recertification scorer under the installed policyengine-us. + +On a three-household H5 in two states, scored with real engines: + +- every household-batched total is the same at batch sizes 1 and 1,000, so + the scored metric set is additive over households for this fixture (a + check of these paths, not a proof for every household); +- the stated-by-derived filing-status table and the role comparison count + every tax unit and person exactly once; +- the counterfactual withholds exactly the newly read columns. + +The wrapper package is not in this environment, so the runtime identities +are stubbed; the real loader replay runs in the qualification environment. +""" + +from __future__ import annotations + +import json + +import pandas as pd +import pytest + +from microcosm.data import compatibility_recertification as recert +from microcosm.data import source_enrichment +from test_support.microcosm_data.compatibility_recertification import ( + CRITICAL_METADATA, + FIXTURE_PRODUCER, + load_tool, +) + + +def _write_h5(path): + person = pd.DataFrame( + { + "person_id": [1, 2, 3, 4, 5, 6], + "person_household_id": [1, 1, 1, 2, 3, 3], + "person_tax_unit_id": [1, 1, 1, 2, 3, 4], + "person_spm_unit_id": [1, 1, 1, 2, 3, 3], + "person_family_id": [1, 1, 1, 2, 3, 3], + "person_marital_unit_id": [1, 1, 2, 3, 4, 5], + "age": [40.0, 38.0, 6.0, 70.0, 30.0, 25.0], + "employment_income": [30_000.0, 12_000.0, 0.0, 0.0, 60_000.0, 8_000.0], + "social_security": [0.0, 0.0, 0.0, 22_000.0, 0.0, 0.0], + "tax_unit_role_input": [ + "HEAD", + "SPOUSE", + "DEPENDENT", + "HEAD", + "HEAD", + "HEAD", + ], + } + ) + tables = { + "person": person, + "household": pd.DataFrame( + { + "household_id": [1, 2, 3], + "household_weight": [100.0, 50.0, 80.0], + "state_fips": [6, 36, 6], + "county_fips": [6037, 36061, 6037], + } + ), + "tax_unit": pd.DataFrame( + { + "tax_unit_id": [1, 2, 3, 4], + "filing_status_input": ["JOINT", "SINGLE", "SINGLE", "SINGLE"], + } + ), + "spm_unit": pd.DataFrame({"spm_unit_id": [1, 2, 3]}), + "family": pd.DataFrame({"family_id": [1, 2, 3]}), + "marital_unit": pd.DataFrame({"marital_unit_id": [1, 2, 3, 4, 5]}), + } + with pd.HDFStore(path, "w") as store: + store.put("_time_period", pd.Series([2024]), format="table") + for key, frame in tables.items(): + store.put(key, frame, format="table", data_columns=True) + + +@pytest.fixture +def scorable(tmp_path, monkeypatch): + artifact_root = tmp_path / "artifacts" + artifact_root.mkdir() + h5 = artifact_root / "populace_us_2024.h5" + _write_h5(h5) + release_dir = tmp_path / "releases" / "populace-us-2024-scoring-fixture" + release_dir.mkdir(parents=True) + dataset = { + "artifact": "populace_us_2024", + "filename": h5.name, + "sha256": recert.sha256_file(h5), + } + (release_dir / recert.REPORT_FILE).write_text(json.dumps({"dataset": dataset})) + targets = [ + { + "name": name, + "metadata": metadata, + "registry": {"family": name.split(".", 1)[0]}, + "target": 1.0, + "final_estimate": 1.0, + } + for name, metadata in CRITICAL_METADATA.items() + ] + for target in targets: + if target["name"].startswith( + "irs_soi.ty2022.historic_table_2.us.all.income_tax" + ): + target["metadata"]["target_role"] = "federal_income_tax_total" + (release_dir / "calibration_diagnostics.json").write_text( + json.dumps( + { + "targets": targets, + "build": {"post_export_scoring": {"spm": {"geography_kind": "county"}}}, + } + ) + ) + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + monkeypatch.setattr( + source_enrichment, + "_runtime_package_identities", + lambda wheels, require_wheels: { + name: {"version": "0", "source_sha256": "0" * 64} + for name in recert.COMPATIBILITY_PACKAGES + }, + ) + return tool, release_dir, artifact_root + + +def test_household_batched_totals_do_not_depend_on_the_batch_size(scorable, tmp_path): + tool, release_dir, artifact_root = scorable + documents = [ + tool.score_candidate( + release_dir=release_dir, + artifact_root=artifact_root, + output=tmp_path / f"scores-{size}.json", + variant=recert.VARIANT_AS_STORED, + batch_households=size, + ) + for size in (1, 1_000) + ] + one, whole = (document["metrics"] for document in documents) + assert set(one) == set(whole) + for metric_id in one: + assert one[metric_id]["value"] == pytest.approx( + whole[metric_id]["value"], rel=1e-12, abs=1e-6 + ), metric_id + # The fixture reaches real engine output. + assert whole["income_tax_net"]["value"] != 0 + assert whole["social_security_total"]["value"] == pytest.approx(50 * 22_000.0) + statuses = documents[1]["filing_status"] + assert ( + sum(cell["tax_units"] for row in statuses.values() for cell in row.values()) + == 4 + ) + assert documents[1]["tax_unit_roles"]["persons"] == 6 + assert documents[0]["scoring"]["passes"][0]["batches"] == 3 + + +def test_the_counterfactual_withholds_exactly_the_newly_read_columns( + scorable, tmp_path +): + from microcosm.data import stored_inputs + + tool, release_dir, artifact_root = scorable + tables = stored_inputs.h5_stored_tables(artifact_root / "populace_us_2024.h5") + changes = recert.stored_column_changes( + tables, + engine_variables=stored_inputs.installed_us_engine().variables, + stored_input_register=stored_inputs.US_STORED_NON_VARIABLE_COLUMNS, + ) + read = sorted(item["column"] for item in changes[recert.CHANGE_READ]) + if not read: + with pytest.raises(ValueError, match="newly reads no stored column"): + tool.score_candidate( + release_dir=release_dir, + artifact_root=artifact_root, + output=tmp_path / "counterfactual.json", + variant=recert.VARIANT_WITHOUT_NEWLY_READ, + ) + return + document = tool.score_candidate( + release_dir=release_dir, + artifact_root=artifact_root, + output=tmp_path / "counterfactual.json", + variant=recert.VARIANT_WITHOUT_NEWLY_READ, + ) + assert document["variant"] == { + "id": recert.VARIANT_WITHOUT_NEWLY_READ, + "withheld_columns": read, + } From eaee621fdcd151bb725f42a8dd3b6bb8e06055db Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 14:00:32 -0400 Subject: [PATCH 06/12] Print a recertification's claim measurement at preflight and publication Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/data/publish_cli.py | 12 +++++-- .../test_compatibility_recertification.py | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-data/src/microcosm/data/publish_cli.py b/packages/microcosm-data/src/microcosm/data/publish_cli.py index 94a0613bf..e41c9f172 100644 --- a/packages/microcosm-data/src/microcosm/data/publish_cli.py +++ b/packages/microcosm-data/src/microcosm/data/publish_cli.py @@ -344,8 +344,16 @@ def main(argv: list[str] | None = None) -> int: file=sys.stderr, ) # A declared compatibility range admits model versions no check measured. - # The bundle records which one was; say it on the same two paths. - measurement = recorded_claim_measurement(Path(args.release_dir)) + # The bundle records which one was; say it on the same two paths. A + # recertification records the measured version, its baseline and the + # drift summary the declarer read. + from microcosm.data import compatibility_recertification + + measurement = recorded_claim_measurement( + Path(args.release_dir) + ) or compatibility_recertification.recorded_claim_measurement( + Path(args.release_dir) + ) if measurement: print( "note: this release declares a publisher compatibility range that " diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index e1d126a0c..ae1f43206 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -744,3 +744,34 @@ def test_prepare_release_takes_wheels_but_not_a_parent_h5( tag_only=True, ) assert "populace_us_2024.h5" in prepared.root_artifacts + + +def test_preflight_prints_the_recertification_measurement( + candidate, tmp_path, monkeypatch, capsys +): + from microcosm.data.publish_cli import main as publish_main + + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) + monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) + wheel = tmp_path / "policyengine_us-2.60.4-py3-none-any.whl" + wheel.write_bytes(b"") + status = publish_main( + [ + str(output), + "--repo-id", + "policyengine/populace-us", + "--artifact-root", + str(artifact_root), + "--compatibility-wheel", + str(wheel), + "--no-latest", + "--tag-only", + "--preflight-only", + ] + ) + captured = capsys.readouterr() + assert status == 0, captured.err + verdict = json.loads(captured.out.strip().splitlines()[-1]) + assert verdict["claim_measurement"]["model"]["measured_versions"] == ["2.60.4"] + assert "measured against only the versions recorded here" in captured.err From 68007a7f2bae002afa569f8b91cbf688ecd80d71 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 15:09:00 -0400 Subject: [PATCH 07/12] Address Sol review r1 on the recertification lane - Bind every score and the loader receipt to every installed byte of the four packages (parameter YAML included), proven against the wheels, and the wheels against PyPI's published digests; drop install-path-dependent direct_url from the identity. - Refuse a nonpositive batch size; require each score to cover every household of the H5 in both passes. - Authenticate artifact placement against the parent tag at publication (LFS sha256 or git blob id), so a root artifact moved into the release directory cannot publish at the wrong path. - Restrict model claims to ==T or >=L, --- docs/us-compatibility-recertification.md | 54 +- .../data/compatibility_recertification.py | 462 ++++++++++++++++-- .../src/microcosm/data/release.py | 7 +- .../test_compatibility_recertification.py | 423 ++++++++++++---- ...t_compatibility_recertification_scoring.py | 24 +- .../compatibility_recertification.py | 132 ++++- .../build_us_compatibility_recertification.py | 89 +++- 7 files changed, 1001 insertions(+), 190 deletions(-) diff --git a/docs/us-compatibility-recertification.md b/docs/us-compatibility-recertification.md index f465c8d0f..42911aded 100644 --- a/docs/us-compatibility-recertification.md +++ b/docs/us-compatibility-recertification.md @@ -33,10 +33,21 @@ every parent artifact, so it is the whole lineage. Offline, the validator: the child's revision; - replays today's release contract on the rebuilt parent (`parent_contract`). -Publication adds the online anchor: after the Hub client exists and before -any branch, commit or upload, `publish_release` downloads both manifests at -the parent's tag and refuses unless they equal the carried bytes -(`require_parent_publication`). Preflight stays offline. +Publication adds the online anchor (`require_parent_publication`). After the +Hub client exists, and before any branch, commit or upload, `publish_release` +checks two things at the parent's tag: + +- both parent manifests are byte for byte the carried ones; +- each parent artifact is published exactly where the child will publish it, + with the pinned digest: each release file under `releases//`, each + root artifact at its root path. It compares an LFS file's SHA-256, or the + git blob id of the carried bytes. + +Offline, which artifacts count as release files rests on where the files sit +locally. This check is what stops a root artifact moved into the release +directory from publishing at the wrong path. Preflight stays offline. A +parent artifact key the child uses for its own evidence (`build_manifest`, +`compatibility_drift`, ...) is refused outright. The parent must be a US calibration release (`release_type` absent or `calibration`) with `dataset_role` `national_default`. A recertification of a @@ -53,7 +64,10 @@ register, under both engines: the baseline score records the built-with engine's verdict and the loader replay grades the tested engine. Such a parent passes as `passed_with_stored_input_evidence_replaced`, the replaced failure lines are recorded verbatim, and certification requires the baseline verdict -to pass. Any other contract failure refuses the child. +to pass. The substitution covers absent evidence only. It applies when the +parent carries no stored-input evidence anywhere and its two manifests agree +on every other `gate_evidence` field. Any other contract failure refuses the +child. ## What the newer engine reads differently @@ -110,9 +124,18 @@ Three score runs feed one drift document: `other_engine_changes_effect`. The drift document embeds all three, and validation recomputes every row from -them. The tested runs must be the exact runtime the loader replay certified -(version and installed-source digest of all four packages). The baseline run -must be the parent's built-with policyengine-us and policyengine-core. +them. Every score document must: + +- cover all households of the H5, in both its household and its state pass; +- score the carried `calibration_diagnostics.json` (by digest) under the + parent's post-export SPM selection; +- run on a runtime proven against the four wheels, every installed byte + included (parameter YAML, not only Python), each wheel's SHA-256 matching + the digest PyPI publishes for that version. + +The tested runs must be exactly the runtime the loader replay certified: +version, content digest and wheel digest of all four packages. The baseline +run must be the parent's built-with policyengine-us and policyengine-core. ## Steps @@ -152,6 +175,10 @@ recorded commit. --variant without_newly_read_inputs --output counterfactual.json ``` + Each run also takes the four `--compatibility-wheel` files its environment + was installed from. Before building any engine, the scorer proves the + install against them and checks them against PyPI. + 4. **Put the drift in front of the declarer.** Whether drift blocks the claim is the declarer's call, not this lane's. Queue it with `cos add-decision`. @@ -172,9 +199,14 @@ recorded commit. ``` The certified manifest keeps every parent entry. It adds one - `publisher_claim` model entry, which must contain the tested version, pass - the source-enrichment claim guards and not reach back to the built-with - version. It also adds the tested Core as an exact `publisher_claim` pin, + `publisher_claim` model entry in one of two bounded shapes: + + - `==T`, the tested version; or + - `>=L, None: path.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") +#: dist-info files an installer writes, which no wheel carries: they say how a +#: package was installed, not what it is (``uv_cache.json`` is uv's). +_INSTALLER_FILES = frozenset( + {"RECORD", "INSTALLER", "REQUESTED", "direct_url.json", "uv_cache.json"} +) + + +def _identity_file(name: str) -> bool: + """Whether an installed or wheel path belongs to the package's identity. + + Bytecode caches and installer bookkeeping are excluded; so are paths the + installer generates outside the package (console scripts, ``../bin``). + Everything else counts, parameter YAML and data files included, because + every byte of it can change what the engine computes. + """ + + parts = name.split("/") + if ".." in parts or "__pycache__" in parts or name.endswith(".pyc"): + return False + return not (parts[0].endswith(".dist-info") and parts[-1] in _INSTALLER_FILES) + + +def _canonical_package(name: str) -> str: + return name.lower().replace("_", "-").replace(".", "-") + + +def runtime_identities( + compatibility_wheels: Sequence[Path | str] = (), *, require_wheels: bool +) -> dict: + """Every installed byte of the four packages, bound to their wheels. + + For each of :data:`COMPATIBILITY_PACKAGES`, the digest covers every file + the distribution installed (``_identity_file``), parameters and data as + well as Python. With a wheel for the package, the installed inventory + must equal the wheel's members and every installed file the member's + bytes, so ``content_sha256`` then identifies the wheel's contents and + ``wheel_sha256`` the wheel itself. ``require_wheels`` demands all four + wheels and refuses an editable install. Nothing here depends on where a + wheel was installed from, so honest replays in two environments agree. + + Raises: + ValueError: A wheel is unknown, duplicated, or differs from the + installed package; or ``require_wheels`` and a wheel is missing. + """ + + from email.parser import BytesParser + from importlib import metadata + from zipfile import ZipFile + + wheels: dict[str, tuple[Path, str, dict[str, bytes]]] = {} + for raw in compatibility_wheels: + path = Path(raw) + with ZipFile(path) as archive: + members = { + name: archive.read(name) + for name in archive.namelist() + if not name.endswith("/") and _identity_file(name) + } + metadata_files = [ + name for name in members if name.endswith(".dist-info/METADATA") + ] + if len(metadata_files) != 1: + raise ValueError(f"{path.name} must contain one wheel METADATA file") + parsed = BytesParser().parsebytes(members[metadata_files[0]]) + name = _canonical_package(parsed["Name"]) + if name not in COMPATIBILITY_PACKAGES or name in wheels: + raise ValueError( + "compatibility wheels must uniquely name the country, Core, wrapper " + "and calculator packages" + ) + wheels[name] = (path, parsed["Version"], members) + if require_wheels and set(wheels) != set(COMPATIBILITY_PACKAGES): + raise ValueError( + "the runtime must be proven against all four wheels: " + f"{', '.join(COMPATIBILITY_PACKAGES)}" + ) + identities = {} + for name in COMPATIBILITY_PACKAGES: + dist = metadata.distribution(name) + direct_url = dist.read_text("direct_url.json") + if ( + require_wheels + and direct_url + and _mapping(_mapping(json.loads(direct_url)).get("dir_info")).get( + "editable" + ) + ): + raise ValueError(f"{name} is an editable install, not a wheel identity") + files = sorted( + str(file) for file in dist.files or () if _identity_file(str(file)) + ) + if not files: + raise ValueError(f"cannot list the installed files of {name}") + digest = hashlib.sha256() + for relative in files: + digest.update(relative.encode() + b"\0") + digest.update(Path(dist.locate_file(relative)).read_bytes() + b"\0") + identity = { + "version": dist.version, + "files": len(files), + "content_sha256": digest.hexdigest(), + "wheel_sha256": None, + } + if name in wheels: + path, version, members = wheels[name] + if version != dist.version: + raise ValueError( + f"{name} wheel {version} is not the installed {dist.version}" + ) + if set(members) != set(files): + raise ValueError( + f"{name} installed files differ from its wheel's members" + ) + for relative in files: + if Path(dist.locate_file(relative)).read_bytes() != members[relative]: + raise ValueError( + f"{name} installed {relative} differs from its wheel" + ) + identity["wheel_sha256"] = sha256_file(path) + identities[name] = identity + return identities + + def is_recertification(release_dir: Path | str) -> bool: """Whether ``release_dir``'s manifest declares this release type. @@ -381,7 +504,14 @@ def describe_parent( artifacts (the H5 and the calibration NPZ). Every artifact the manifest declares must be present with the digest it pins, in exactly one of the two places: a bare filename in the release directory is a release file, - anything else a root artifact. + anything else a root artifact. Offline, that placement rests on where the + files sit locally; publication authenticates it against the parent tag on + the Hub (:func:`require_parent_publication`), so a root artifact moved into + the release directory cannot publish at the wrong path. + + A parent artifact key that the child uses for its own evidence is refused: + the child's entry would replace it, and the artifact would silently leave + the child's manifest. Raises: ValueError: The parent is not a supported published release, or a @@ -416,6 +546,11 @@ def describe_parent( ) release_files, root_artifacts = {}, {} for key, raw in _mapping(manifest.get("artifacts")).items(): + if key in CHILD_ARTIFACT_KEYS or key in CERTIFIED_ARTIFACT_KEYS: + raise ValueError( + f"parent artifact key {key!r} is one the child writes its own " + "evidence under; the parent artifact would leave the child manifest" + ) entry = _mapping(raw) path, sha = entry.get("path"), entry.get("sha256") if not isinstance(path, str) or not isinstance(sha, str): @@ -507,13 +642,22 @@ def replay_parent_contract( itself, against the current register, under the built-with engine (the baseline scores) and the tested engine (the loader replay), so such a parent passes as :data:`PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED`, - with the replaced failure lines recorded. Any other failure is returned in - ``other_failures`` and refuses the child. + with the replaced failure lines recorded. + + The substitution covers absent evidence and nothing else. It applies only + when the parent carries no stored-input evidence anywhere (no evidence + file, artifact, ``gate_evidence`` entry or manifest block) and its two + manifests agree on every other ``gate_evidence`` field; the evidence check + also compares those blocks whole, so without the second condition an + unrelated gate disagreement would be excused with the missing evidence. + Otherwise every failure is returned in ``other_failures`` and refuses the + child. """ from microcosm.data import contract release_dir = Path(release_dir) + key = contract.US_STORED_INPUTS_EVIDENCE_KEY with tempfile.TemporaryDirectory(prefix=".recertification-parent-") as staging: parent_dir = _reconstructed_parent(release_dir, parent, Path(staging)) try: @@ -531,8 +675,26 @@ def replay_parent_contract( failures=evidence_failures, evidence_tier=False, ) - replaced = [line for line in failures if line in evidence_failures] - other = [line for line in failures if line not in evidence_failures] + evidence_file_present = ( + parent_dir / contract.US_STORED_INPUTS_EVIDENCE_FILE + ).exists() + manifest_build = _mapping(manifest.get("build")) + gate_blocks = [ + _mapping(holder.get("gate_evidence")) + for holder in (_mapping(build), manifest_build) + ] + evidence_absent = ( + not evidence_file_present + and key not in _mapping(manifest.get("artifacts")) + and all(key not in holder for holder in (_mapping(build), manifest_build)) + and all(key not in block for block in gate_blocks) + ) + other_gates_agree = gate_blocks[0] == gate_blocks[1] + replaceable = ( + set(evidence_failures) if evidence_absent and other_gates_agree else set() + ) + replaced = [line for line in failures if line in replaceable] + other = [line for line in failures if line not in replaceable] return { "status": PARENT_CONTRACT_PASSED if not failures @@ -576,8 +738,8 @@ def run_recertification_loader_replay( The receipt this returns is replayed at every later gate, so it holds only what is deterministic for the H5 and the installed runtime: - 1. The four packages' installed sources, bound to the four wheels when - ``require_wheels`` (``source_enrichment._runtime_package_identities``). + 1. Every installed byte of the four packages, parameters included, bound + to the four wheels when ``require_wheels`` (:func:`runtime_identities`). 2. The stored-input classification (:func:`stored_column_changes`) and the register digests. An unreviewed change refuses the replay. 3. Both native loaders (country ``USSingleYearDataset``, wrapper @@ -603,9 +765,7 @@ def run_recertification_loader_replay( from microcosm.data import source_enrichment, stored_inputs candidate_h5 = Path(candidate_h5) - packages = source_enrichment._runtime_package_identities( - compatibility_wheels, require_wheels=require_wheels - ) + packages = runtime_identities(compatibility_wheels, require_wheels=require_wheels) tables = stored_inputs.h5_stored_tables(candidate_h5) engine = stored_inputs.installed_us_engine() changes = stored_column_changes( @@ -1123,12 +1283,72 @@ def _critical_tolerances() -> dict[str, float]: VARIANT_WITHOUT_NEWLY_READ = "without_newly_read_inputs" +#: Recorded by the tool's scorer; bump with any change to how it scores. +SCORING_METHOD = "household_batched_country_loader" + + def _score_document_failures( - scores: Mapping, *, dataset_sha256: str, definitions: Mapping, label: str + scores: Mapping, + *, + dataset_sha256: str, + definitions: Mapping, + label: str, + diagnostics_sha256: str, + spm: object, + expected_households: int, ) -> list[str]: - """Shape and binding checks for one embedded score document.""" + """Shape and binding checks for one embedded score document. + + Beyond shape: the document scored the carried parent diagnostics' metric + definitions under the parent's own post-export SPM selection, covered + every household of the H5 in each of its passes, and ran on a runtime + whose every package was proven against a wheel whose digest PyPI + publishes for that version. + """ failures = [] + if scores.get("parent_diagnostics_sha256") != diagnostics_sha256: + failures.append(f"{label} scores must score the carried parent diagnostics") + scoring = _mapping(scores.get("scoring")) + if ( + scoring.get("method") != SCORING_METHOD + or scoring.get("period") != SCORING_PERIOD + or scoring.get("spm") != spm + ): + failures.append( + f"{label} scores must use the {SCORING_METHOD} method for " + f"{SCORING_PERIOD} under the parent's post-export SPM selection {spm!r}" + ) + batch = scoring.get("batch_households") + if type(batch) is not int or batch <= 0: + failures.append(f"{label} scores must record a positive batch size") + passes = scoring.get("passes") + if ( + scoring.get("n_households") != expected_households + or not isinstance(passes, list) + or {_mapping(item).get("batching") for item in passes} + != {BATCHING_HOUSEHOLD, BATCHING_STATE} + or any( + _mapping(item).get("households") != expected_households for item in passes + ) + ): + failures.append( + f"{label} scores must cover all {expected_households} households of " + "the H5 in both their household and their state passes" + ) + packages = _mapping(_mapping(scores.get("runtime")).get("packages")) + for package in COMPATIBILITY_PACKAGES: + identity = _mapping(packages.get(package)) + if not ( + isinstance(identity.get("version"), str) + and isinstance(identity.get("content_sha256"), str) + and isinstance(identity.get("wheel_sha256"), str) + and identity.get("pypi_sha256_verified") is True + ): + failures.append( + f"{label} scores must run {package} proven against a wheel whose " + "digest PyPI publishes for its version" + ) if ( scores.get("schema_version") != SCORES_SCHEMA_VERSION or scores.get("kind") != SCORES_KIND @@ -1176,20 +1396,32 @@ def combine_drift( baseline_engine: Mapping, tested_packages: Mapping, newly_read_columns: Sequence[str], + diagnostics_sha256: str, + expected_households: int, ) -> dict: """Combine score documents into the drift document, or raise ``ValueError``. ``baseline`` must have been scored under the parent's built-with policyengine-us and policyengine-core (``baseline_engine``); ``tested`` - under exactly ``tested_packages`` (every package's version and installed - source digest, as the loader replay recorded them). ``counterfactual``, if - given, must be the tested runtime with exactly ``newly_read_columns`` - withheld, and is required whenever there are any. Every row recomputes - from the embedded documents. + under exactly ``tested_packages`` (every package's version, installed + content digest and wheel digest, as the loader replay recorded them). + ``counterfactual``, if given, must be the tested runtime with exactly + ``newly_read_columns`` withheld, and is required whenever there are any. + Every document must score the carried diagnostics (``diagnostics_sha256``) + under the parent's SPM selection and cover all ``expected_households`` + (:func:`_score_document_failures`). Every row recomputes from the + embedded documents. """ definitions = drift_metric_definitions(calibration_diagnostics) + spm = _mapping( + _mapping(calibration_diagnostics.get("build")).get("post_export_scoring") + ).get("spm") failures = [] + if not isinstance(spm, Mapping): + failures.append( + "the parent diagnostics record no post-export SPM selection to score under" + ) exact = set(_exact_requirement_ids()) scored_exact = { metric["requirement_id"] @@ -1222,6 +1454,9 @@ def combine_drift( dataset_sha256=dataset_sha256, definitions=definitions, label=label, + diagnostics_sha256=diagnostics_sha256, + spm=spm, + expected_households=expected_households, ) ) recorded_variant = _mapping(scores.get("variant")) @@ -1244,9 +1479,9 @@ def combine_drift( for package in COMPATIBILITY_PACKAGES: want = _mapping(tested_packages.get(package)) got = _mapping(observed.get(package)) - if (got.get("version"), got.get("source_sha256")) != ( - want.get("version"), - want.get("source_sha256"), + if any( + got.get(field) != want.get(field) + for field in ("version", "content_sha256", "wheel_sha256") ): failures.append( f"{label} scores must run the certified {package} runtime" @@ -1398,30 +1633,105 @@ def drift_from_scores( drift: Mapping, *, calibration_diagnostics: Mapping, + dataset_sha256: str, + diagnostics_sha256: str, + expected_households: int, baseline_engine: Mapping, tested_packages: Mapping, newly_read_columns: Sequence[str], ) -> dict: - """Recompute a drift document from the score documents it embeds.""" + """Recompute a drift document from the score documents it embeds. + + The dataset, diagnostics and household count come from the release being + validated, never from the drift document, so a drift cannot vouch for + itself. + """ scores = _mapping(drift.get("scores")) return combine_drift( calibration_diagnostics=calibration_diagnostics, - dataset_sha256=str(drift.get("dataset_sha256")), + dataset_sha256=dataset_sha256, baseline=_mapping(scores.get("baseline")), tested=_mapping(scores.get("tested")), counterfactual=scores.get("counterfactual"), baseline_engine=baseline_engine, tested_packages=tested_packages, newly_read_columns=newly_read_columns, + diagnostics_sha256=diagnostics_sha256, + expected_households=expected_households, ) +def h5_household_count(path: Path | str) -> int: + """Households in a US release H5, from HDF metadata alone.""" + + import h5py + + with h5py.File(path, "r") as h5: + return int(h5["household/table"].shape[0]) + + # --------------------------------------------------------------------------- # The claim. # --------------------------------------------------------------------------- +def _check_recertification_claim( + specifier: object, *, tested: object, built_with: object +) -> None: + """Refuse a model claim outside this lane's grammar. + + The source-enrichment guards probe a claim's bounds; they cannot prove + them (``>=2.60.4,!=3.0.0,!=99999.0.0`` passes them and admits 4.0). A + recertification therefore takes only two shapes, both bounded by + construction: + + - ``==T``, the tested version exactly; or + - ``>=L,="] + and not tested_version.is_prerelease + and bounds[1] <= tested_version < bounds[0] <= next_minor + and bounds[1] > built_version + ): + return + raise ValueError( + f"publisher compatibility claim {specifier!r} is not '=={tested_version}' " + f"or '>=L, str: + """The id git (and so the Hub) gives a file of ``content``'s bytes.""" + + return hashlib.sha1(b"blob %d\0" % len(content) + content).hexdigest() + + def require_parent_publication( - api: object, release_dir: Path | str, *, repo_id: str + api: object, + release_dir: Path | str, + *, + repo_id: str, + artifact_root: Path | str | None, ) -> None: - """Refuse unless the parent tag on the Hub serves the carried manifests. - - Runs after the publisher constructs its Hub client and before any upload. - The parent's ``release_manifest.json`` pins every parent artifact, so equal - bytes at the parent's immutable tag mean the lineage the child carries is - the published one. Absence and difference both refuse; a transient Hub - failure propagates. + """Refuse unless the parent tag on the Hub serves what the child carries. + + Runs after the publisher constructs its Hub client and before any branch, + commit or upload. Two checks, both against the parent's immutable tag: + + - the carried ``release_manifest.json`` and ``build_manifest.json`` are + byte for byte the parent's published ones. The release manifest pins + every parent artifact's digest, so this anchors the whole lineage; + - every parent artifact sits on the Hub where the child will publish it: + each carried release file under ``releases//`` and each root + artifact at its root path, with the pinned digest (an LFS file's + SHA-256, or the git blob id of the carried bytes). Offline, which + artifacts are release files rests on where the files sit locally; this + is what stops a root artifact moved into the release directory from + publishing at the wrong path. + + Absence and difference both refuse. A transient Hub failure propagates. """ from microcosm.data.release import _is_absent_entry_error, _is_absent_revision_error @@ -2130,6 +2456,50 @@ def require_parent_publication( f"{carried}; refusing to publish a child of a parent the Hub does " "not serve." ) + get_paths_info = getattr(api, "get_paths_info", None) + if not callable(get_paths_info): + raise TypeError( + "publishing a recertification needs a Hub client with get_paths_info, " + "to authenticate where the parent's artifacts are published" + ) + expected = {} + for name, sha in _mapping(parent.get("release_files")).items(): + expected[f"releases/{parent_id}/{name}"] = (sha, release_dir / name) + root = Path(artifact_root) if artifact_root is not None else None + for path, sha in _mapping(parent.get("root_artifacts")).items(): + expected[path] = (sha, None if root is None else root / path) + published = { + getattr(info, "path", None): info + for info in get_paths_info( + repo_id=repo_id, + paths=sorted(expected), + revision=revision, + repo_type="dataset", + ) + } + for path_in_repo, (sha, local) in sorted(expected.items()): + info = published.get(path_in_repo) + if info is None: + raise ValueError( + f"parent {parent_id!r} publishes no {path_in_repo} at tag " + f"{revision!r}; the child would publish an artifact the parent " + "does not have there" + ) + lfs = getattr(info, "lfs", None) + if lfs is not None: + matches = getattr(lfs, "sha256", None) == sha + else: + matches = ( + local is not None + and local.is_file() + and sha256_file(local) == sha + and getattr(info, "blob_id", None) == git_blob_sha1(local.read_bytes()) + ) + if not matches: + raise ValueError( + f"{path_in_repo} at tag {revision!r} is not the artifact the child " + "carries; refusing to publish it there" + ) # --------------------------------------------------------------------------- diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index 0cbd2b453..d336e320a 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -678,7 +678,12 @@ def publish_release( # The offline gates proved the child against the parent manifest it # carries; only the Hub can say that manifest is the published one. # Checked before any branch, commit or upload. - require_parent_publication(api, prepared.release_dir, repo_id=repo_id) + require_parent_publication( + api, + prepared.release_dir, + repo_id=repo_id, + artifact_root=prepared.artifact_root, + ) if evidence: payload = latest_evidence_pointer_payload( release_id, diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index ae1f43206..3e5bba305 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -242,6 +242,22 @@ def test_describe_parent_refuses_unsupported_parents(parent, update, message): recert.describe_parent(release_dir, artifact_root) +@pytest.mark.parametrize("key", ["build_manifest", "compatibility_drift"]) +def test_describe_parent_refuses_a_key_the_child_writes_its_evidence_under(parent, key): + """Sol r1 finding 6: the child's entry would silently replace the artifact.""" + + release_dir, artifact_root = parent + + def rename(manifest): + manifest["artifacts"][key] = manifest["artifacts"].pop( + "populace_us_2024_calibration" + ) + + _rewrite_json(release_dir / "release_manifest.json", rename) + with pytest.raises(ValueError, match="child writes its own"): + recert.describe_parent(release_dir, artifact_root) + + def test_describe_parent_refuses_an_artifact_that_differs_from_its_pin(parent): release_dir, artifact_root = parent (artifact_root / "populace_us_2024.h5").write_bytes(b"other bytes") @@ -293,6 +309,38 @@ def strip_release(manifest): _rewrite_json(release_dir / "release_manifest.json", strip_release) +def test_the_substitution_does_not_excuse_an_unrelated_gate_disagreement( + parent, tmp_path, monkeypatch +): + """Sol r1 finding 7: only absent stored-input evidence is replaceable.""" + + release_dir, artifact_root = parent + _strip_stored_input_evidence(release_dir) + _rewrite_json( + release_dir / "build_manifest.json", + lambda build: build.setdefault("gate_evidence", {}).update( + reform_coverage_smoke="skipped" + ), + ) + _rewrite_json( + release_dir / "release_manifest.json", + lambda manifest: ( + manifest["build"] + .setdefault("gate_evidence", {}) + .update(reform_coverage_smoke="bound") + ), + ) + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + with pytest.raises(ValueError, match="different gate_evidence blocks"): + tool.build_candidate( + parent_release_dir=release_dir, + parent_artifact_root=artifact_root, + output_dir=tmp_path / "out", + release_id=CHILD_ID, + ) + + def test_a_parent_without_stored_input_evidence_is_replaced_not_waived( parent, tmp_path, monkeypatch ): @@ -436,15 +484,20 @@ def test_claim_keeps_every_parent_entry_and_adds_the_measured_versions(candidate @pytest.mark.parametrize( - "specifier, message", + "specifier", [ - (">=1.0,<2.61", "reaches back to the built-with"), - (">=2.60.4", "reaches 3.0.0"), - ("==2.60.3", "excludes the tested"), + ">=1.0,<2.61", + ">=2.60.4", + "==2.60.3", + ">=2.60.4,!=3.0.0,!=99999.0.0", + ">=2.60.4,<2.62", + "~=2.60.4", + "==2.60.*", + ">2.60.3,<2.61", ], ) -def test_claims_that_overreach_are_refused(candidate, specifier, message): - with pytest.raises(ValueError, match=message): +def test_claims_outside_the_bounded_grammar_are_refused(candidate, specifier): + with pytest.raises(ValueError, match="is not '==2.60.4'"): recert.claim_entries( _parent_manifest(candidate), tested_packages=TESTED_PACKAGES, @@ -453,6 +506,17 @@ def test_claims_that_overreach_are_refused(candidate, specifier, message): ) +@pytest.mark.parametrize("specifier", ["==2.60.4", ">=2.60.0,<2.61", ">=2.60.4,<2.61"]) +def test_claims_inside_the_bounded_grammar_are_accepted(candidate, specifier): + models, _, claims = recert.claim_entries( + _parent_manifest(candidate), + tested_packages=TESTED_PACKAGES, + specifier=specifier, + declared_by=DECLARER, + ) + assert claims["model"]["specifier"] == specifier == models[-1]["specifier"] + + def test_a_parent_that_already_covers_the_tested_version_needs_no_child(candidate): packages = {**TESTED_PACKAGES, "policyengine-us": {"version": "1.729.0"}} with pytest.raises(ValueError, match="nothing to recertify"): @@ -528,45 +592,83 @@ def test_tampered_certified_children_are_refused( validate_release_dir(output, artifact_root=artifact_root) -def test_drift_refuses_a_baseline_on_another_engine(candidate): - release_dir, _ = candidate +def _combine(release_dir, **overrides): diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) report = json.loads((release_dir / recert.REPORT_FILE).read_text()) - baseline = score_document(release_dir, packages=TESTED_PACKAGES) + arguments = { + "calibration_diagnostics": diagnostics, + "dataset_sha256": report["dataset"]["sha256"], + "baseline": score_document(release_dir, packages=BASELINE_PACKAGES), + "tested": score_document(release_dir), + "counterfactual": None, + "baseline_engine": { + "model": {"version": "1.729.0"}, + "core": {"version": "3.19.0"}, + }, + "tested_packages": TESTED_PACKAGES, + "newly_read_columns": [], + "diagnostics_sha256": recert.sha256_file( + release_dir / "calibration_diagnostics.json" + ), + "expected_households": HOUSEHOLDS, + } + arguments.update(overrides) + return recert.combine_drift(**arguments) + + +def test_drift_refuses_a_baseline_on_another_engine(candidate): + release_dir, _ = candidate with pytest.raises(ValueError, match="built-with policyengine-us"): - recert.combine_drift( - calibration_diagnostics=diagnostics, - dataset_sha256=report["dataset"]["sha256"], - baseline=baseline, - tested=score_document(release_dir), - counterfactual=None, - baseline_engine={ - "model": {"version": "1.729.0"}, - "core": {"version": "3.19.0"}, - }, - tested_packages=TESTED_PACKAGES, - newly_read_columns=[], - ) + _combine(release_dir, baseline=score_document(release_dir)) def test_drift_requires_the_counterfactual_when_columns_are_newly_read(candidate): release_dir, _ = candidate - diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) - report = json.loads((release_dir / recert.REPORT_FILE).read_text()) with pytest.raises(ValueError, match="counterfactual"): - recert.combine_drift( - calibration_diagnostics=diagnostics, - dataset_sha256=report["dataset"]["sha256"], - baseline=score_document(release_dir, packages=BASELINE_PACKAGES), - tested=score_document(release_dir), - counterfactual=None, - baseline_engine={ - "model": {"version": "1.729.0"}, - "core": {"version": "3.19.0"}, - }, - tested_packages=TESTED_PACKAGES, - newly_read_columns=["tax_unit_role_input"], - ) + _combine(release_dir, newly_read_columns=["tax_unit_role_input"]) + + +@pytest.mark.parametrize( + "role, document_overrides, message", + [ + ("tested", {"households": 2}, "cover all 3 households"), + ("baseline", {"households": 0}, "cover all 3 households"), + ("tested", {"spm": {"geography_kind": "state"}}, "post-export SPM selection"), + ], +) +def test_drift_refuses_scores_that_did_not_cover_or_bind_the_parent( + candidate, role, document_overrides, message +): + release_dir, _ = candidate + packages = BASELINE_PACKAGES if role == "baseline" else TESTED_PACKAGES + document = score_document(release_dir, packages=packages, **document_overrides) + with pytest.raises(ValueError, match=message): + _combine(release_dir, **{role: document}) + + +def test_drift_refuses_scores_of_other_diagnostics(candidate): + release_dir, _ = candidate + with pytest.raises(ValueError, match="carried parent diagnostics"): + _combine(release_dir, diagnostics_sha256="0" * 64) + + +@pytest.mark.parametrize("field", ["content_sha256", "wheel_sha256"]) +def test_drift_refuses_a_tested_runtime_the_replay_did_not_certify(candidate, field): + release_dir, _ = candidate + packages = { + **TESTED_PACKAGES, + "policyengine-us": {**TESTED_PACKAGES["policyengine-us"], field: "f" * 64}, + } + with pytest.raises(ValueError, match="certified policyengine-us runtime"): + _combine(release_dir, tested=score_document(release_dir, packages=packages)) + + +def test_drift_refuses_a_runtime_not_proven_against_pypi(candidate): + release_dir, _ = candidate + document = score_document(release_dir, packages=BASELINE_PACKAGES) + document["runtime"]["packages"]["policyengine-us"]["pypi_sha256_verified"] = False + with pytest.raises(ValueError, match="PyPI publishes"): + _combine(release_dir, baseline=document) @settings( @@ -579,44 +681,20 @@ def test_drift_requires_the_counterfactual_when_columns_are_newly_read(candidate new=st.floats(-1e12, 1e12, allow_nan=False), withheld=st.floats(-1e12, 1e12, allow_nan=False), ) -def test_drift_rows_decompose_exactly(parent, base, new, withheld): - release_dir, _ = parent - diagnostics = json.loads((release_dir / "calibration_diagnostics.json").read_text()) - definitions = recert.drift_metric_definitions(diagnostics) - dataset = {"artifact": "x", "filename": "x.h5", "sha256": "c" * 64} - - def document( - packages, value, variant=recert.VARIANT_AS_STORED, withheld_columns=() - ): - return { - "schema_version": recert.SCORES_SCHEMA_VERSION, - "kind": recert.SCORES_KIND, - "dataset": dataset, - "variant": {"id": variant, "withheld_columns": list(withheld_columns)}, - "metric_definitions_sha256": recert.canonical_sha256(definitions), - "runtime": {"packages": packages}, - "metrics": { - m["metric_id"]: {"value": value} for m in definitions["metrics"] - }, - "code": dict(FIXTURE_PRODUCER), - } - - drift = recert.combine_drift( - calibration_diagnostics=diagnostics, - dataset_sha256=dataset["sha256"], - baseline=document(BASELINE_PACKAGES, base), - tested=document(TESTED_PACKAGES, new), - counterfactual=document( - TESTED_PACKAGES, - withheld, - recert.VARIANT_WITHOUT_NEWLY_READ, - ["parent_1_id"], +def test_drift_rows_decompose_exactly(candidate, base, new, withheld): + release_dir, _ = candidate + drift = _combine( + release_dir, + baseline=score_document( + release_dir, packages=BASELINE_PACKAGES, value=lambda metric_id: base + ), + tested=score_document(release_dir, value=lambda metric_id: new), + counterfactual=score_document( + release_dir, + variant=recert.VARIANT_WITHOUT_NEWLY_READ, + withheld=("parent_1_id",), + value=lambda metric_id: withheld, ), - baseline_engine={ - "model": {"version": "1.729.0"}, - "core": {"version": "3.19.0"}, - }, - tested_packages=TESTED_PACKAGES, newly_read_columns=["parent_1_id"], ) for row in drift["rows"]: @@ -652,19 +730,6 @@ def test_signed_component_scores_soi_rows_as_the_release_tool_does(): # --------------------------------------------------------------------------- -def _seed_parent(hub, parent_dir): - tag = parent_dir.name - hub._commits["commit-parent"] = { - f"releases/{tag}/release_manifest.json": ( - parent_dir / "release_manifest.json" - ).read_bytes(), - f"releases/{tag}/build_manifest.json": ( - parent_dir / "build_manifest.json" - ).read_bytes(), - } - hub._refs[tag] = "commit-parent" - - def _publish(hub, output, artifact_root, monkeypatch): monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) @@ -681,43 +746,94 @@ def _publish(hub, output, artifact_root, monkeypatch): def test_publication_refuses_a_parent_the_hub_does_not_serve( - hub, parent, candidate, tmp_path, monkeypatch + paths_hub, parent, candidate, tmp_path, monkeypatch ): output, artifact_root = certify(candidate, tmp_path, monkeypatch) with pytest.raises(ValueError, match="is not published"): - _publish(hub, output, artifact_root, monkeypatch) - assert hub.events == [] + _publish(paths_hub, output, artifact_root, monkeypatch) + assert paths_hub.events == [] def test_publication_refuses_a_parent_whose_published_manifest_differs( - hub, parent, candidate, tmp_path, monkeypatch + paths_hub, parent, candidate, tmp_path, monkeypatch ): - parent_dir, _ = parent + parent_dir, parent_root = parent output, artifact_root = certify(candidate, tmp_path, monkeypatch) - _seed_parent(hub, parent_dir) - hub._commits["commit-parent"][ + seed_published_parent(paths_hub, parent_dir, parent_root) + paths_hub._commits["commit-parent"][ f"releases/{parent_dir.name}/release_manifest.json" ] = b"{}" with pytest.raises(ValueError, match="differs from the carried"): + _publish(paths_hub, output, artifact_root, monkeypatch) + assert paths_hub.events == [] + + +def test_publication_needs_a_hub_that_can_say_where_artifacts_are( + hub, parent, candidate, tmp_path, monkeypatch +): + parent_dir, parent_root = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + seed_published_parent(hub, parent_dir, parent_root) + with pytest.raises(TypeError, match="get_paths_info"): _publish(hub, output, artifact_root, monkeypatch) assert hub.events == [] +def test_publication_refuses_a_root_artifact_moved_into_the_release_directory( + paths_hub, parent, candidate, tmp_path, monkeypatch +): + """Sol r1 finding 3: an unchanged root NPZ reclassified as a release file.""" + + parent_dir, parent_root = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + npz = "populace_us_2024_calibration.npz" + shutil.copyfile(artifact_root / npz, output / npz) + os.chmod(output / npz, 0o400) + + def reclassify(report): + sha = report["parent"]["root_artifacts"].pop(npz) + report["parent"]["release_files"][npz] = sha + + _rewrite_json(output / recert.REPORT_FILE, reclassify) + _rewrite_json( + output / "release_manifest.json", + lambda manifest: manifest["artifacts"]["compatibility_recertification"].update( + sha256=recert.sha256_file(output / recert.REPORT_FILE) + ), + ) + seed_published_parent(paths_hub, parent_dir, parent_root) + with pytest.raises(ValueError, match="publishes no"): + _publish(paths_hub, output, artifact_root, monkeypatch) + assert paths_hub.events == [] + + +def test_publication_refuses_a_root_artifact_the_parent_tag_holds_other_bytes_of( + paths_hub, parent, candidate, tmp_path, monkeypatch +): + parent_dir, parent_root = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + seed_published_parent(paths_hub, parent_dir, parent_root) + paths_hub._commits["commit-parent"]["populace_us_2024.h5"] = b"other bytes" + with pytest.raises(ValueError, match="is not the artifact the child carries"): + _publish(paths_hub, output, artifact_root, monkeypatch) + assert paths_hub.events == [] + + def test_publication_of_a_published_parents_child_is_tag_only( - hub, parent, candidate, tmp_path, monkeypatch + paths_hub, parent, candidate, tmp_path, monkeypatch ): - parent_dir, _ = parent + parent_dir, parent_root = parent output, artifact_root = certify(candidate, tmp_path, monkeypatch) - _seed_parent(hub, parent_dir) - _publish(hub, output, artifact_root, monkeypatch) - uploads = dict(hub.uploads) + seed_published_parent(paths_hub, parent_dir, parent_root) + _publish(paths_hub, output, artifact_root, monkeypatch) + uploads = dict(paths_hub.uploads) assert ( uploads["populace_us_2024.h5"] == (artifact_root / "populace_us_2024.h5").read_bytes() ) assert f"releases/{CHILD_ID}/{recert.DRIFT_FILE}" in uploads assert "latest.json" not in uploads - assert [tag["tag"] for tag in hub.tags] == [CHILD_ID] + assert [tag["tag"] for tag in paths_hub.tags] == [CHILD_ID] def test_prepare_release_takes_wheels_but_not_a_parent_h5( @@ -775,3 +891,110 @@ def test_preflight_prints_the_recertification_measurement( verdict = json.loads(captured.out.strip().splitlines()[-1]) assert verdict["claim_measurement"]["model"]["measured_versions"] == ["2.60.4"] assert "measured against only the versions recorded here" in captured.err + + +# --------------------------------------------------------------------------- +# Runtime identity (Sol r1 findings 1 and 8). +# --------------------------------------------------------------------------- + + +def _fake_install(root, name, version): + """An installed distribution and its wheel, sharing every package byte.""" + + from zipfile import ZipFile + + module = name.replace("-", "_") + dist_info = f"{module}-{version}.dist-info" + files = { + f"{module}/__init__.py": b"VALUE = 1\n", + f"{module}/parameters/rate.yaml": b"values:\n 2024-01-01: 0.1\n", + f"{dist_info}/METADATA": ( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + ), + } + site = root / "site" + for relative, content in files.items(): + (site / relative).parent.mkdir(parents=True, exist_ok=True) + (site / relative).write_bytes(content) + (site / dist_info / "uv_cache.json").write_text("{}") + record = [*files, f"{dist_info}/uv_cache.json", f"{dist_info}/RECORD"] + (site / dist_info / "RECORD").write_text("".join(f"{path},,\n" for path in record)) + wheel = root / f"{module}-{version}-py3-none-any.whl" + with ZipFile(wheel, "w") as archive: + for relative, content in files.items(): + archive.writestr(relative, content) + archive.writestr(f"{dist_info}/RECORD", "") + return site / dist_info, wheel + + +@pytest.fixture +def fake_runtime(tmp_path, monkeypatch): + from importlib import metadata + + installs = { + name: _fake_install(tmp_path, name, version) + for name, version in ( + ("policyengine-us", "2.60.4"), + ("policyengine-core", "3.33.3"), + ("policyengine", "6.10.1"), + ("spm-calculator", "1.0.0"), + ) + } + monkeypatch.setattr( + metadata, + "distribution", + lambda name: metadata.PathDistribution(installs[name][0]), + ) + return tmp_path / "site", tuple(wheel for _, wheel in installs.values()) + + +def test_runtime_identity_covers_parameters_and_ignores_installer_files(fake_runtime): + site, wheels = fake_runtime + identities = recert.runtime_identities(wheels, require_wheels=True) + assert identities["policyengine-us"]["files"] == 3 + assert identities["policyengine-us"]["wheel_sha256"] == recert.sha256_file( + wheels[0] + ) + assert "direct_url" not in identities["policyengine-us"] + before = identities["policyengine-us"]["content_sha256"] + (site / "policyengine_us-2.60.4.dist-info" / "uv_cache.json").write_text('{"x": 1}') + assert ( + recert.runtime_identities(wheels, require_wheels=True)["policyengine-us"][ + "content_sha256" + ] + == before + ) + + +def test_a_patched_parameter_file_is_not_the_wheels_runtime(fake_runtime): + site, wheels = fake_runtime + (site / "policyengine_us" / "parameters" / "rate.yaml").write_bytes( + b"values:\n 2024-01-01: 0.2\n" + ) + with pytest.raises(ValueError, match="rate.yaml differs from its wheel"): + recert.runtime_identities(wheels, require_wheels=True) + unbound = recert.runtime_identities((), require_wheels=False) + assert unbound["policyengine-us"]["wheel_sha256"] is None + + +def test_the_runtime_needs_all_four_wheels(fake_runtime): + _, wheels = fake_runtime + with pytest.raises(ValueError, match="all four wheels"): + recert.runtime_identities(wheels[:3], require_wheels=True) + + +def test_the_scorer_refuses_a_nonpositive_batch_size_before_any_work( + candidate, tmp_path +): + tool = load_tool() + release_dir, artifact_root = candidate + for size in (0, -1): + with pytest.raises(ValueError, match="positive integer"): + tool.score_candidate( + release_dir=release_dir, + artifact_root=artifact_root, + output=tmp_path / f"scores-{size}.json", + variant=recert.VARIANT_AS_STORED, + compatibility_wheels=(), + batch_households=size, + ) diff --git a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py index 1e3de32e5..06ee31e73 100644 --- a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py +++ b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py @@ -9,8 +9,9 @@ every tax unit and person exactly once; - the counterfactual withholds exactly the newly read columns. -The wrapper package is not in this environment, so the runtime identities -are stubbed; the real loader replay runs in the qualification environment. +The wrapper package is not in this environment and no wheels are at hand, +so the scorer's runtime proof is stubbed; the real loader replay and wheel +proof run in the qualification environments. """ from __future__ import annotations @@ -21,7 +22,6 @@ import pytest from microcosm.data import compatibility_recertification as recert -from microcosm.data import source_enrichment from test_support.microcosm_data.compatibility_recertification import ( CRITICAL_METADATA, FIXTURE_PRODUCER, @@ -117,10 +117,16 @@ def scorable(tmp_path, monkeypatch): tool = load_tool() monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) monkeypatch.setattr( - source_enrichment, - "_runtime_package_identities", - lambda wheels, require_wheels: { - name: {"version": "0", "source_sha256": "0" * 64} + tool, + "scoring_runtime", + lambda wheels: { + name: { + "version": "0", + "files": 0, + "content_sha256": "0" * 64, + "wheel_sha256": "0" * 64, + "pypi_sha256_verified": True, + } for name in recert.COMPATIBILITY_PACKAGES }, ) @@ -135,6 +141,7 @@ def test_household_batched_totals_do_not_depend_on_the_batch_size(scorable, tmp_ artifact_root=artifact_root, output=tmp_path / f"scores-{size}.json", variant=recert.VARIANT_AS_STORED, + compatibility_wheels=(), batch_households=size, ) for size in (1, 1_000) @@ -155,6 +162,7 @@ def test_household_batched_totals_do_not_depend_on_the_batch_size(scorable, tmp_ ) assert documents[1]["tax_unit_roles"]["persons"] == 6 assert documents[0]["scoring"]["passes"][0]["batches"] == 3 + assert [item["households"] for item in documents[0]["scoring"]["passes"]] == [3, 3] def test_the_counterfactual_withholds_exactly_the_newly_read_columns( @@ -177,6 +185,7 @@ def test_the_counterfactual_withholds_exactly_the_newly_read_columns( artifact_root=artifact_root, output=tmp_path / "counterfactual.json", variant=recert.VARIANT_WITHOUT_NEWLY_READ, + compatibility_wheels=(), ) return document = tool.score_candidate( @@ -184,6 +193,7 @@ def test_the_counterfactual_withholds_exactly_the_newly_read_columns( artifact_root=artifact_root, output=tmp_path / "counterfactual.json", variant=recert.VARIANT_WITHOUT_NEWLY_READ, + compatibility_wheels=(), ) assert document["variant"] == { "id": recert.VARIANT_WITHOUT_NEWLY_READ, diff --git a/test_support/microcosm_data/compatibility_recertification.py b/test_support/microcosm_data/compatibility_recertification.py index 0ba639c12..fffab75a4 100644 --- a/test_support/microcosm_data/compatibility_recertification.py +++ b/test_support/microcosm_data/compatibility_recertification.py @@ -25,18 +25,62 @@ from test_support.microcosm_data.release import _bind_stored_input_evidence CHILD_ID = "populace-us-2024-9f1260b-recert-pe-us-2.60.4" + + +def _identity(version, digit): + return { + "version": version, + "files": 10, + "content_sha256": digit * 64, + "wheel_sha256": chr(ord(digit) + 1) * 64, + } + + +#: Receipt-shaped identities (no PyPI flag; the replay does not check PyPI). TESTED_PACKAGES = { - "policyengine-us": {"version": "2.60.4", "source_sha256": "1" * 64}, - "policyengine-core": {"version": "3.33.3", "source_sha256": "2" * 64}, - "policyengine": {"version": "6.10.1", "source_sha256": "3" * 64}, - "spm-calculator": {"version": "1.0.0", "source_sha256": "4" * 64}, + "policyengine-us": _identity("2.60.4", "1"), + "policyengine-core": _identity("3.33.3", "3"), + "policyengine": _identity("6.10.1", "5"), + "spm-calculator": _identity("1.0.0", "7"), } BASELINE_PACKAGES = { - "policyengine-us": {"version": "1.729.0", "source_sha256": "5" * 64}, - "policyengine-core": {"version": "3.19.0", "source_sha256": "6" * 64}, - "policyengine": {"version": "6.10.1", "source_sha256": "3" * 64}, - "spm-calculator": {"version": "1.0.0", "source_sha256": "4" * 64}, + "policyengine-us": _identity("1.729.0", "a"), + "policyengine-core": _identity("3.19.0", "c"), + "policyengine": _identity("6.10.1", "5"), + "spm-calculator": _identity("1.0.0", "7"), } +HOUSEHOLDS = 3 + + +def scored_runtime(packages): + """Score-document identities: the receipt's plus the PyPI verification.""" + + return { + name: {**identity, "pypi_sha256_verified": True} + for name, identity in packages.items() + } + + +def write_fixture_h5(path: Path, households: int = HOUSEHOLDS) -> None: + """A real HDF5 file whose metadata names ``households`` households. + + Engine-free tests never load it; validation reads only its household + count from metadata, and digests its bytes. + """ + + import h5py + import numpy as np + + with h5py.File(path, "w") as h5: + h5.create_dataset( + "household/table", + data=np.array( + [(index, index + 1) for index in range(households)], + dtype=[("index", " None: h5 = artifact_root / "populace_us_2024.h5" npz = artifact_root / "populace_us_2024_calibration.npz" - h5.write_bytes(b"synthetic parent microdata, not an HDF5 file") + write_fixture_h5(h5) npz.write_bytes(b"synthetic parent calibration weights") diagnostics = release_dir / "calibration_diagnostics.json" @@ -242,6 +286,8 @@ def score_document( withheld=(), value=lambda metric_id: 1_000.0, stored_input_passed=True, + households=HOUSEHOLDS, + spm=None, ) -> dict: """A score document shaped as the tool writes it, with invented values.""" @@ -256,7 +302,21 @@ def score_document( "dataset": report["dataset"], "variant": {"id": variant, "withheld_columns": sorted(withheld)}, "metric_definitions_sha256": recert.canonical_sha256(definitions), - "runtime": {"packages": packages}, + "parent_diagnostics_sha256": recert.sha256_file( + release_dir / "calibration_diagnostics.json" + ), + "runtime": {"packages": scored_runtime(packages)}, + "scoring": { + "method": recert.SCORING_METHOD, + "period": recert.SCORING_PERIOD, + "spm": spm or diagnostics["build"]["post_export_scoring"]["spm"], + "batch_households": 2_000, + "n_households": HOUSEHOLDS, + "passes": [ + {"batching": recert.BATCHING_HOUSEHOLD, "households": households}, + {"batching": recert.BATCHING_STATE, "households": households}, + ], + }, "engine": { "label": f"policyengine-us {packages['policyengine-us']['version']}", "stored_input_verdict": { @@ -303,6 +363,58 @@ def write_scores(directory: Path, release_dir: Path, **overrides) -> dict[str, P return paths +class PathsInfoHub(FakeHub): + """FakeHub with ``get_paths_info``: LFS for H5/NPZ, git blob ids otherwise.""" + + def get_paths_info(self, *, repo_id, paths, revision=None, repo_type=None): + from types import SimpleNamespace + + assert repo_type == "dataset" and repo_id == self.repo_id + files = self._commits[self._refs.get(revision or "main", revision)] + infos = [] + for path in paths: + if path not in files: + continue + content = files[path] + lfs = ( + SimpleNamespace(sha256=hashlib.sha256(content).hexdigest()) + if path.endswith((".h5", ".npz")) + else None + ) + infos.append( + SimpleNamespace( + path=path, lfs=lfs, blob_id=recert.git_blob_sha1(content) + ) + ) + return infos + + +def seed_published_parent(hub, parent_dir: Path, artifact_root: Path) -> None: + """Publish the parent's files at its tag on the fake Hub, as the publisher did.""" + + tag = parent_dir.name + manifest = json.loads((parent_dir / "release_manifest.json").read_text()) + files = { + f"releases/{tag}/{name}": (parent_dir / name).read_bytes() + for name in ("release_manifest.json", "build_manifest.json") + } + for entry in manifest["artifacts"].values(): + path = entry["path"] + if (parent_dir / path).is_file(): + files[f"releases/{tag}/{path}"] = (parent_dir / path).read_bytes() + else: + files[path] = (artifact_root / path).read_bytes() + hub._commits["commit-parent"] = files + hub._refs[tag] = "commit-parent" + + +@pytest.fixture +def paths_hub(tmp_path) -> PathsInfoHub: + fake = PathsInfoHub() + fake._download_dir = tmp_path / "hub-cache" + return fake + + def certify(candidate, tmp_path, monkeypatch, *, specifier="policyengine-us==2.60.4"): """Certify the candidate with the fake replay and fixture scores.""" diff --git a/tools/build_us_compatibility_recertification.py b/tools/build_us_compatibility_recertification.py index ee0d46641..8b40e892d 100644 --- a/tools/build_us_compatibility_recertification.py +++ b/tools/build_us_compatibility_recertification.py @@ -519,16 +519,68 @@ def roles(self, counts): ) +def _pypi_wheel_sha256(name: str, version: str, filename: str) -> str | None: + """The SHA-256 PyPI publishes for one wheel file, or ``None`` if it has none.""" + + from urllib.request import urlopen + + with urlopen( + f"https://pypi.org/pypi/{name}/{version}/json", timeout=60 + ) as response: + payload = json.load(response) + for item in payload.get("urls", ()): + if item.get("filename") == filename: + return (item.get("digests") or {}).get("sha256") + return None + + +def scoring_runtime(compatibility_wheels: Sequence[Path]) -> dict: + """The four packages' identities, each wheel checked against PyPI's digest. + + ``runtime_identities`` proves every installed byte against its wheel; + this adds that the wheel is the one PyPI publishes for that version, so a + score cannot rest on a locally patched engine bearing a released version. + """ + + identities = recert.runtime_identities(compatibility_wheels, require_wheels=True) + by_package = {} + for wheel in compatibility_wheels: + path = Path(wheel) + for name, identity in identities.items(): + if identity["wheel_sha256"] == recert.sha256_file(path): + by_package[name] = path + for name, identity in identities.items(): + path = by_package[name] + published = _pypi_wheel_sha256(name, identity["version"], path.name) + identity["pypi_sha256_verified"] = published == identity["wheel_sha256"] + if not identity["pypi_sha256_verified"]: + raise ValueError( + f"{path.name} is not the wheel PyPI publishes for {name} " + f"{identity['version']}" + ) + return identities + + def score_candidate( *, release_dir: Path, artifact_root: Path, output: Path, variant: str, + compatibility_wheels: Sequence[Path], batch_households: int = _DEFAULT_BATCH_HOUSEHOLDS, microsimulation_cls=None, ) -> dict: - """Score the candidate's H5 under the installed engine; write the document.""" + """Score the candidate's H5 under the installed engine; write the document. + + The runtime is proven against the four wheels, and they against PyPI, + before any engine is built, so a long run cannot end on an unprovable + runtime. + """ + + if type(batch_households) is not int or batch_households <= 0: + raise ValueError("--batch-households must be a positive integer") + runtime = scoring_runtime(compatibility_wheels) import numpy as np from policyengine_us.data import USSingleYearDataset @@ -600,7 +652,7 @@ def score_candidate( ), ): selected = [m for m in metrics.values() if m["batching"] == batching] - count = 0 + count = scored = 0 for ids in batches: batch = frames.batch(ids) simulation = microsimulation_cls( @@ -628,28 +680,27 @@ def score_candidate( "scored in batches (microcosm#956)." ) count += 1 + scored += len(ids) del simulation, scorer, batch gc.collect() passes.append( - {"batching": batching, "batches": count, "metrics": len(selected)} + { + "batching": batching, + "batches": count, + "households": scored, + "metrics": len(selected), + } ) - packages = source_enrichment._runtime_package_identities((), require_wheels=False) document = { "schema_version": recert.SCORES_SCHEMA_VERSION, "kind": recert.SCORES_KIND, "dataset": dict(dataset), "variant": {"id": variant, "withheld_columns": withheld}, "metric_definitions_sha256": recert.canonical_sha256(definitions), - "runtime": { - "python": platform.python_version(), - "packages": { - name: { - "version": identity["version"], - "source_sha256": identity["source_sha256"], - } - for name, identity in packages.items() - }, - }, + "parent_diagnostics_sha256": recert.sha256_file( + release_dir / "calibration_diagnostics.json" + ), + "runtime": {"python": platform.python_version(), "packages": runtime}, "engine": { "label": engine.label, "stored_input_verdict": { @@ -664,7 +715,7 @@ def score_candidate( "column_changes": changes, }, "scoring": { - "method": "household_batched_country_loader", + "method": recert.SCORING_METHOD, "period": recert.SCORING_PERIOD, "spm": spm, "batch_households": batch_households, @@ -709,6 +760,13 @@ def main(argv: Sequence[str] | None = None) -> int: score.add_argument( "--batch-households", type=int, default=_DEFAULT_BATCH_HOUSEHOLDS ) + score.add_argument( + "--compatibility-wheel", + action="append", + type=Path, + required=True, + help="the installed country, Core, wrapper and calculator wheels (all four)", + ) args = parser.parse_args(argv) try: if args.command == "fetch-parent": @@ -743,6 +801,7 @@ def main(argv: Sequence[str] | None = None) -> int: artifact_root=args.artifact_root, output=args.output, variant=args.variant, + compatibility_wheels=tuple(args.compatibility_wheel), batch_households=args.batch_households, ) print( From 3a89a047c64908b93e865f4256dd442988ce2d2f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 15:49:18 -0400 Subject: [PATCH 08/12] Address Sol review r2 and drop the wrapper from the proven runtime - policyengine.py certifies its bundled data release against the installed country model at import: policyengine 6.10.1 under policyengine-us 2.60.4 raises before any loader exists (found by the real-file rehearsal). The replay now proves country, Core and calculator, and records the wrapper loader as not attested; the downstream pe.py certification proves it. - Runtime proof: files on disk under each package must equal the recorded inventory (no unrecorded modules); imports must resolve to the installed package; the engine compiles from source into an isolated, empty bytecode cache, and every loaded module is checked against the verified tree after the run. - Placement: refuse parent paths the publisher would not write (releases/, unclean paths); refuse a parent tag that publishes an artifact at both locations; check offline that the publisher's upload plan covers every carried artifact. - Stored-input substitution compares gate_evidence blocks as written. - Docs: carry what the manifest declares, not every file on the tag. Co-Authored-By: Claude Opus 5.5 --- .../us-compatibility-recertification.added.md | 2 +- docs/us-compatibility-recertification.md | 67 +++- .../data/compatibility_recertification.py | 335 +++++++++++++++--- .../test_compatibility_recertification.py | 149 +++++++- ...t_compatibility_recertification_scoring.py | 3 + .../compatibility_recertification.py | 2 - .../build_us_compatibility_recertification.py | 14 +- 7 files changed, 487 insertions(+), 85 deletions(-) diff --git a/changelog.d/us-compatibility-recertification.added.md b/changelog.d/us-compatibility-recertification.added.md index f79c2855a..3936e49f0 100644 --- a/changelog.d/us-compatibility-recertification.added.md +++ b/changelog.d/us-compatibility-recertification.added.md @@ -1 +1 @@ -Add the `compatibility_recertification` release type: a published US calibration release republished byte for byte with a measured `compatible_model_packages` claim for a newer policyengine-us. It carries the parent's published manifests as its identity, replays the native loaders under the four tested wheels, records the engine's reviewed column changes in both directions, and embeds the drift of the critical rows and headline totals between the built-with and tested engines. Publication refuses a child whose parent tag does not serve the carried manifest bytes. `tools/build_us_compatibility_recertification.py` fetches the parent, builds the candidate and scores it; see `docs/us-compatibility-recertification.md`. +Add the `compatibility_recertification` release type: a published US calibration release republished byte for byte with a measured `compatible_model_packages` claim for a newer policyengine-us. It carries the parent's published manifests as its identity, replays the country loader under the tested country, Core and calculator wheels (compiled from verified source into an isolated bytecode cache), records the engine's reviewed column changes in both directions, and embeds the drift of the critical rows and headline totals between the built-with and tested engines. Publication refuses a child whose parent tag does not serve the carried manifest bytes. `tools/build_us_compatibility_recertification.py` fetches the parent, builds the candidate and scores it; see `docs/us-compatibility-recertification.md`. diff --git a/docs/us-compatibility-recertification.md b/docs/us-compatibility-recertification.md index 42911aded..cbb563e0f 100644 --- a/docs/us-compatibility-recertification.md +++ b/docs/us-compatibility-recertification.md @@ -2,13 +2,14 @@ A `compatibility_recertification` release lets a published, certified US calibration release carry a measured `compatible_model_packages` claim for a -newer policyengine-us without a rebuild. The child republishes the parent's -H5, calibration weights and every release file byte for byte under a new id -and tag. It transforms no data and never recalibrates. What it adds is -evidence about the newer engine: - -- **a loader replay** under the four tested wheels (country, Core, wrapper, - calculator); +newer policyengine-us without a rebuild. The child republishes byte for byte, +under a new id and tag, every artifact the parent's release manifest declares +(the H5, the calibration weights and every declared release file) and both +parent manifests. A file the parent tag carries without a manifest entry is +not bound by the manifest and is not carried. The child transforms no data +and never recalibrates. What it adds is evidence about the newer engine: + +- **a loader replay** under the tested country, Core and calculator wheels; - **the engine's column changes**: what the newer engine ignores that the built-with one read, and what it reads that no engine read before; - **the drift**: the 18 exact critical rows, the scorable national SOI Table @@ -16,6 +17,16 @@ evidence about the newer engine: engines; - **a claim**, declared by a named person after reading the drift. +**The wrapper loader is not attested here.** policyengine.py imports its +bundled country model at `import policyengine`, and certifies the bundle's +data release against the installed country version as it does. +policyengine 6.10.1 under policyengine-us 2.60.4 raises "Data release +manifest is not certified for the runtime model version 2.60.4" before any +loader exists. So the first wrapper that can load this child is one whose +bundle certifies it, which is the downstream policyengine.py certification +step; that step runs the wrapper's own suite on the pinned pair. The receipt +records `wrapper_loader: not_attested` with this reason. + The module is `microcosm.data.compatibility_recertification`; the tool is `tools/build_us_compatibility_recertification.py`. @@ -45,9 +56,18 @@ checks two things at the parent's tag: Offline, which artifacts count as release files rests on where the files sit locally. This check is what stops a root artifact moved into the release -directory from publishing at the wrong path. Preflight stays offline. A -parent artifact key the child uses for its own evidence (`build_manifest`, -`compatibility_drift`, ...) is refused outright. +directory from publishing at the wrong path. Each artifact must sit at exactly +one of the two places: a parent that publishes a file at both its root and its +release-file location is refused as ambiguous. Offline, the validator also +replays the publisher's own upload plan on the child directory, and requires it +to write every carried artifact to its manifest path. + +Preflight stays offline. A parent artifact is refused outright when: + +- its key is one the child uses for its own evidence (`build_manifest`, + `compatibility_drift`, ...); or +- its path is not one the publisher writes (anything under `releases/`, or + not a clean relative path). The parent must be a US calibration release (`release_type` absent or `calibration`) with `dataset_role` `national_default`. A recertification of a @@ -66,8 +86,8 @@ passes as `passed_with_stored_input_evidence_replaced`, the replaced failure lines are recorded verbatim, and certification requires the baseline verdict to pass. The substitution covers absent evidence only. It applies when the parent carries no stored-input evidence anywhere and its two manifests agree -on every other `gate_evidence` field. Any other contract failure refuses the -child. +on every other `gate_evidence` field (compared as written: `[]` and `{}` +disagree). Any other contract failure refuses the child. ## What the newer engine reads differently @@ -129,12 +149,21 @@ them. Every score document must: - cover all households of the H5, in both its household and its state pass; - score the carried `calibration_diagnostics.json` (by digest) under the parent's post-export SPM selection; -- run on a runtime proven against the four wheels, every installed byte - included (parameter YAML, not only Python), each wheel's SHA-256 matching - the digest PyPI publishes for that version. +- run on a runtime proven against the three wheels: + - every installed file is included (parameter YAML, not only Python); + - the files on disk under each package are exactly the recorded ones, so an + unrecorded module cannot ride along; + - each import resolves to the installed package, not to something earlier + on the path; + - each wheel's SHA-256 matches the digest PyPI publishes for that version. + +The scorer and the loader replay also compile the packages from source into a +fresh, empty bytecode cache (`sys.pycache_prefix`), so a replaced `.pyc` in the +install tree is never run. After the run they check that every loaded module +came from the verified tree and was compiled into that cache. The tested runs must be exactly the runtime the loader replay certified: -version, content digest and wheel digest of all four packages. The baseline +version, content digest and wheel digest of all three packages. The baseline run must be the parent's built-with policyengine-us and policyengine-core. ## Steps @@ -164,7 +193,7 @@ recorded commit. 3. **Score** in two isolated environments, each with this checkout's `microcosm-data` installed editable. One has the parent's built-with - wheels, the other the four tested wheels: + wheels, the other the three tested wheels: ```bash BASE/bin/python tools/build_us_compatibility_recertification.py score \ @@ -175,7 +204,7 @@ recorded commit. --variant without_newly_read_inputs --output counterfactual.json ``` - Each run also takes the four `--compatibility-wheel` files its environment + Each run also takes the three `--compatibility-wheel` files its environment was installed from. Before building any engine, the scorer proves the install against them and checks them against PyPI. @@ -216,7 +245,7 @@ recorded commit. microcosm-publish-release /path/certified/releases/CHILD_ID \ --repo-id policyengine/populace-us \ --artifact-root /path/candidate/artifacts \ - --compatibility-wheel ... (four) \ + --compatibility-wheel ... (three) \ --no-latest --tag-only --preflight-only ``` diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py index 47aa8d79b..9a63e3389 100644 --- a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -57,6 +57,7 @@ import hashlib import json import math +import posixpath import re import shutil import tempfile @@ -113,12 +114,34 @@ "compatibility_recertification_loader": RECEIPT_FILE, "compatibility_drift": DRIFT_FILE, } -COMPATIBILITY_PACKAGES = ( - "policyengine-us", - "policyengine-core", - "policyengine", - "spm-calculator", +#: The runtime a recertification proves: the country model, its Core and the +#: calculator that registers the native SPM role, by import name. +IMPORT_NAMES = MappingProxyType( + { + "policyengine-us": "policyengine_us", + "policyengine-core": "policyengine_core", + "spm-calculator": "spm_calculator", + } ) +COMPATIBILITY_PACKAGES = tuple(IMPORT_NAMES) +#: The wrapper is not part of it. policyengine.py imports its bundled country +#: model at ``import policyengine`` and certifies the bundle's data release +#: against the installed country version there: policyengine 6.10.1 under +#: policyengine-us 2.60.4 raises "Data release manifest is not certified for +#: the runtime model version 2.60.4" before any loader exists. A wrapper that +#: loads this child is one whose bundle certifies this child, which is the +#: downstream policyengine.py certification step; that step runs the +#: wrapper's own suite on the pinned pair. The receipt records this. +WRAPPER_LOADER_STATUS = { + "status": "not_attested", + "package": "policyengine", + "reason": ( + "policyengine.py certifies its bundled data release against the " + "installed country model at import, so no released wrapper imports " + "under a country version its bundle does not certify; the wrapper " + "loader is proven when a policyengine.py bundle certifies this child" + ), +} #: The producer inventory the release assembler, scorer and certifier run #: from. Publication authenticates every file against the recorded commit and #: the checkout (``source_enrichment._check_producer_source_identity``). @@ -375,25 +398,63 @@ def _canonical_package(name: str) -> str: return name.lower().replace("_", "-").replace(".", "-") +def _package_tree_files(dist, identity_files: Sequence[str]) -> set[str]: + """Every file on disk under the distribution's top-level packages. + + ``dist.files`` is the installer's record, not the tree: a module dropped + into an installed package after installation is absent from it yet + importable, and Core loads every variable module it finds. So the tree is + listed from disk and must equal the record (bytecode caches aside). + """ + + roots = { + relative.split("/", 1)[0] + for relative in identity_files + if "/" in relative and not relative.split("/", 1)[0].endswith(".dist-info") + } + found = set() + for root in roots: + base = Path(dist.locate_file(root)) + for path in base.rglob("*"): + if path.is_dir(): + continue + relative = f"{root}/{path.relative_to(base).as_posix()}" + if _identity_file(relative): + found.add(relative) + return found + + def runtime_identities( compatibility_wheels: Sequence[Path | str] = (), *, require_wheels: bool ) -> dict: - """Every installed byte of the four packages, bound to their wheels. - - For each of :data:`COMPATIBILITY_PACKAGES`, the digest covers every file - the distribution installed (``_identity_file``), parameters and data as - well as Python. With a wheel for the package, the installed inventory - must equal the wheel's members and every installed file the member's - bytes, so ``content_sha256`` then identifies the wheel's contents and - ``wheel_sha256`` the wheel itself. ``require_wheels`` demands all four + """The three packages' installed contents, bound to their wheels. + + For each of :data:`COMPATIBILITY_PACKAGES`: + + - the digest covers every file the distribution installed + (``_identity_file``), parameters and data as well as Python; + - the files on disk under its packages must be exactly the recorded ones, + so an unrecorded module cannot ride along (:func:`_package_tree_files`); + - the package an import resolves to must be the installed one, so nothing + earlier on the path (``PYTHONPATH``, a ``.pth`` entry) shadows it. + + With a wheel for the package, the installed inventory must equal the + wheel's members and every installed file the member's bytes, so + ``content_sha256`` then identifies the wheel's contents and + ``wheel_sha256`` the wheel itself. ``require_wheels`` demands all three wheels and refuses an editable install. Nothing here depends on where a wheel was installed from, so honest replays in two environments agree. + This proves the files; :func:`isolate_bytecode` and + :func:`verify_loaded_modules` prove that what ran was compiled from them. + Raises: ValueError: A wheel is unknown, duplicated, or differs from the - installed package; or ``require_wheels`` and a wheel is missing. + installed package; the tree holds unrecorded files; an import + resolves elsewhere; or ``require_wheels`` and a wheel is missing. """ + import importlib.util from email.parser import BytesParser from importlib import metadata from zipfile import ZipFile @@ -416,13 +477,13 @@ def runtime_identities( name = _canonical_package(parsed["Name"]) if name not in COMPATIBILITY_PACKAGES or name in wheels: raise ValueError( - "compatibility wheels must uniquely name the country, Core, wrapper " - "and calculator packages" + "compatibility wheels must uniquely name the country, Core and " + f"calculator packages ({', '.join(COMPATIBILITY_PACKAGES)})" ) wheels[name] = (path, parsed["Version"], members) if require_wheels and set(wheels) != set(COMPATIBILITY_PACKAGES): raise ValueError( - "the runtime must be proven against all four wheels: " + "the runtime must be proven against all three wheels: " f"{', '.join(COMPATIBILITY_PACKAGES)}" ) identities = {} @@ -442,6 +503,23 @@ def runtime_identities( ) if not files: raise ValueError(f"cannot list the installed files of {name}") + unrecorded = _package_tree_files(dist, files) - set(files) + if unrecorded: + raise ValueError( + f"{name} has files on disk its installation never recorded: " + f"{sorted(unrecorded)[:5]}" + ) + spec = importlib.util.find_spec(IMPORT_NAMES[name]) + installed_root = Path(dist.locate_file(IMPORT_NAMES[name])).resolve() + if ( + spec is None + or spec.origin is None + or not Path(spec.origin).resolve().is_relative_to(installed_root) + ): + raise ValueError( + f"import {IMPORT_NAMES[name]} does not resolve to the installed " + f"{name} at {installed_root}" + ) digest = hashlib.sha256() for relative in files: digest.update(relative.encode() + b"\0") @@ -472,6 +550,95 @@ def runtime_identities( return identities +#: The empty bytecode cache this process compiles the packages into +#: (:func:`isolate_bytecode`); ``None`` until it is created. +_ISOLATED_PYCACHE: str | None = None + + +def _loaded_package_modules() -> list: + import sys + + roots = set(IMPORT_NAMES.values()) + return [ + module + for name, module in list(sys.modules.items()) + if name.split(".", 1)[0] in roots and module is not None + ] + + +def isolate_bytecode() -> str: + """Compile the packages from source into a fresh, empty bytecode cache. + + CPython runs a cached ``.pyc`` whose header matches its source's size and + timestamp without reading the source, so a replaced cache payload would + run code the authenticated source does not contain. Pointing + ``sys.pycache_prefix`` at a new empty directory before the packages are + imported means every module they load is compiled from its source. + Call it before importing any of the packages; it refuses if one is + already loaded under another cache. + + Raises: + RuntimeError: A package module was imported before isolation. + """ + + import sys + + global _ISOLATED_PYCACHE + if _ISOLATED_PYCACHE is not None and sys.pycache_prefix == _ISOLATED_PYCACHE: + return _ISOLATED_PYCACHE + loaded = [module.__name__ for module in _loaded_package_modules()] + if loaded: + raise RuntimeError( + "the country, Core or calculator package was imported before its " + f"bytecode was isolated ({sorted(loaded)[:3]}); run the replay or " + "scorer in a fresh process" + ) + _ISOLATED_PYCACHE = tempfile.mkdtemp(prefix="recertification-pycache-") + sys.pycache_prefix = _ISOLATED_PYCACHE + return _ISOLATED_PYCACHE + + +def verify_loaded_modules() -> int: + """Every loaded module of the packages came from their verified trees. + + Each must have been read from a file :func:`runtime_identities` covers, + under its installed package, and compiled into the isolated cache. Returns + how many modules were checked. + + Raises: + RuntimeError: A module was loaded from elsewhere or from install-tree + bytecode. + """ + + from importlib import metadata + + if _ISOLATED_PYCACHE is None: + raise RuntimeError("isolate_bytecode() must run before the packages load") + roots = { + import_name: Path( + metadata.distribution(package).locate_file(import_name) + ).resolve() + for package, import_name in IMPORT_NAMES.items() + } + prefix = Path(_ISOLATED_PYCACHE).resolve() + checked = 0 + for module in _loaded_package_modules(): + origin = getattr(module, "__file__", None) + if origin is None: + continue + root = roots[module.__name__.split(".", 1)[0]] + cached = getattr(module, "__cached__", None) + if not Path(origin).resolve().is_relative_to(root) or ( + cached is not None and not Path(cached).resolve().is_relative_to(prefix) + ): + raise RuntimeError( + f"{module.__name__} was loaded from {origin} (bytecode {cached}), " + "not compiled from the verified installation" + ) + checked += 1 + return checked + + def is_recertification(release_dir: Path | str) -> bool: """Whether ``release_dir``'s manifest declares this release type. @@ -555,6 +722,20 @@ def describe_parent( path, sha = entry.get("path"), entry.get("sha256") if not isinstance(path, str) or not isinstance(sha, str): raise ValueError(f"parent artifact {key!r} has no path and sha256") + # The publisher writes a release file to releases// and a + # root artifact to its own path; it writes nothing to another + # release's directory. A path it would not resolve to one of those two + # places cannot be republished faithfully, so it is refused. + if ( + path.startswith("/") + or "\\" in path + or posixpath.normpath(path) != path + or path.split("/", 1)[0] == "releases" + ): + raise ValueError( + f"parent artifact {key!r} path {path!r} is not a release file or " + "root artifact path the publisher would write" + ) if entry.get("revision") != parent_id or entry.get("repo_id") != REPO_ID: raise ValueError( f"parent artifact {key!r} must pin {REPO_ID} at the parent tag" @@ -627,6 +808,7 @@ def _reconstructed_parent(release_dir: Path, parent: Mapping, staging: Path) -> #: stored-input evidence, which the recertification re-measures itself under #: both engines (the baseline scores' and the loader replay's verdicts). PARENT_CONTRACT_PASSED = "passed" +_ABSENT = object() PARENT_CONTRACT_STORED_INPUT_EVIDENCE_REPLACED = ( "passed_with_stored_input_evidence_replaced" ) @@ -679,17 +861,22 @@ def replay_parent_contract( parent_dir / contract.US_STORED_INPUTS_EVIDENCE_FILE ).exists() manifest_build = _mapping(manifest.get("build")) - gate_blocks = [ - _mapping(holder.get("gate_evidence")) + raw_blocks = [ + holder.get("gate_evidence", _ABSENT) for holder in (_mapping(build), manifest_build) ] + gate_blocks = [_mapping(block) for block in raw_blocks] evidence_absent = ( not evidence_file_present and key not in _mapping(manifest.get("artifacts")) and all(key not in holder for holder in (_mapping(build), manifest_build)) and all(key not in block for block in gate_blocks) ) - other_gates_agree = gate_blocks[0] == gate_blocks[1] + # Compared as written, not as normalised: [] and {} disagree, as the + # evidence check itself says. + other_gates_agree = raw_blocks[0] == raw_blocks[1] and all( + block is _ABSENT or isinstance(block, Mapping) for block in raw_blocks + ) replaceable = ( set(evidence_failures) if evidence_absent and other_gates_agree else set() ) @@ -727,6 +914,24 @@ def _h5_values(table, column): return values +def _check_source_ownership( + source_paths: Mapping[str, str | None], packages: Mapping[str, str] +) -> None: + """Each named source file lies in the installed tree of its package.""" + + from importlib import metadata + + for label, package in packages.items(): + root = Path( + metadata.distribution(package).locate_file(IMPORT_NAMES[package]) + ).resolve() + source = source_paths.get(label) + if source is None or not Path(source).resolve().is_relative_to(root): + raise ValueError( + f"tested {label} source does not come from the installed {package}" + ) + + def run_recertification_loader_replay( candidate_h5: Path | str, *, @@ -738,14 +943,17 @@ def run_recertification_loader_replay( The receipt this returns is replayed at every later gate, so it holds only what is deterministic for the H5 and the installed runtime: - 1. Every installed byte of the four packages, parameters included, bound - to the four wheels when ``require_wheels`` (:func:`runtime_identities`). + 1. The three packages' installed contents, parameters included, bound to + the three wheels when ``require_wheels`` (:func:`runtime_identities`), + compiled from source into an isolated cache (:func:`isolate_bytecode`) + and, after the run, every loaded module from the verified trees + (:func:`verify_loaded_modules`). 2. The stored-input classification (:func:`stored_column_changes`) and the register digests. An unreviewed change refuses the replay. - 3. Both native loaders (country ``USSingleYearDataset``, wrapper - ``PolicyEngineUSDataset``) return every stored column the tested engine - defines with the H5's exact values, and every id, membership and weight - column too. + 3. The country loader (``USSingleYearDataset``) returns every stored + column the tested engine defines with the H5's exact values, and every + id, membership and weight column too. The wrapper loader is recorded as + not attested (:data:`WRAPPER_LOADER_STATUS`). 4. When the H5 stores the native SPM role, its source comes from the calculator wheel and a complete household's supplied role reaches Core unchanged (``source_enrichment._check_native_input_precedence``). @@ -754,11 +962,12 @@ def run_recertification_loader_replay( step, outside the receipt. """ + isolate_bytecode() + import inspect import h5py import numpy as np - from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset from policyengine_us.data import USSingleYearDataset from policyengine_us.system import system @@ -778,14 +987,8 @@ def run_recertification_loader_replay( raise ValueError(" ".join(failures)) role = source_enrichment.ROLE_VARIABLE stores_role = role in tables.get("person", ()) - source_paths = { - "country_loader": inspect.getsourcefile(USSingleYearDataset), - "wrapper_loader": inspect.getsourcefile(PolicyEngineUSDataset), - } - loaded_source_packages = { - "country_loader": "policyengine-us", - "wrapper_loader": "policyengine", - } + source_paths = {"country_loader": inspect.getsourcefile(USSingleYearDataset)} + loaded_source_packages = {"country_loader": "policyengine-us"} if stores_role: variable = system.variables.get(role) if ( @@ -796,17 +999,8 @@ def run_recertification_loader_replay( raise ValueError(f"tested country must register {role} as a person bool") source_paths["native_role"] = inspect.getsourcefile(type(variable)) loaded_source_packages["native_role"] = "spm-calculator" - if require_wheels: - source_enrichment._check_loaded_source_ownership( - source_paths, loaded_source_packages - ) + _check_source_ownership(source_paths, loaded_source_packages) country = USSingleYearDataset(file_path=str(candidate_h5)) - wrapper = PolicyEngineUSDataset( - name="compatibility_recertification", - description="Read-only compatibility recertification loader replay", - filepath=str(candidate_h5), - year=SCORING_PERIOD, - ) checked = 0 with h5py.File(candidate_h5, "r") as h5: for entity in _ENTITIES: @@ -826,10 +1020,7 @@ def run_recertification_loader_replay( if column not in engine.variables and column not in structural: continue expected = _h5_values(table, column) - for loader, frame in ( - ("country", getattr(country, entity)), - ("wrapper", getattr(wrapper.data, entity)), - ): + for loader, frame in (("country", getattr(country, entity)),): if column not in frame.columns: raise ValueError( f"{loader} native loader dropped {entity}.{column}" @@ -848,13 +1039,15 @@ def run_recertification_loader_replay( checked += 1 checks = [ "country:every_stored_change_reviewed", - "country_and_wrapper:every_engine_variable_column_loaded_unchanged", + "country:every_engine_variable_column_loaded_unchanged", ] if stores_role: source_enrichment._check_native_input_precedence( country, native_inputs={role: "person"} ) checks.append("country:complete_household_native_role_overrides_default") + verify_loaded_modules() + checks.append("runtime:every_loaded_module_compiled_from_the_verified_tree") columns = {column for names in tables.values() for column in names} return { "schema_version": RECEIPT_SCHEMA_VERSION, @@ -867,6 +1060,7 @@ def run_recertification_loader_replay( "source_enrichment": sha256_file(source_enrichment.__file__), }, "packages": packages, + "wrapper_loader": dict(WRAPPER_LOADER_STATUS), "loaded_source_packages": loaded_source_packages, "loaded_source_sha256": { key: sha256_file(path) for key, path in source_paths.items() @@ -1856,7 +2050,7 @@ def validate_recertification_candidate( passed, the loader replay, the claim and the drift, each recomputed. With ``require_compatibility`` (certification output, publish preflight and publication) a pending child is refused, the loader replay must bind the - four wheels, and every producer is authenticated against this checkout. + three wheels, and every producer is authenticated against this checkout. """ from microcosm.data.contract import _check_release_manifest @@ -2001,6 +2195,28 @@ def validate_recertification_candidate( f"child's own; unexpected {sorted(present - expected_files)}, missing " f"{sorted(expected_files - present)}" ) + elif not failures: + # The publisher decides from the same directory what it uploads where. + # Every carried artifact must land at its manifest-resolved path in + # the child: release files under releases//, root artifacts at + # their root paths. + from microcosm.data import release as publisher + + planned_root = publisher._release_manifest_root_artifacts(release_dir) + planned_release = { + *publisher._release_manifest_release_artifacts(release_dir), + *required_release_files(release_id), + } + if ( + planned_root != dict(_mapping(parent.get("root_artifacts"))) + or not set(_mapping(parent.get("release_files"))) <= planned_release + ): + failures.append( + "the publisher would not upload every carried artifact to its " + "manifest path: root " + f"{sorted(planned_root)} against " + f"{sorted(_mapping(parent.get('root_artifacts')))}" + ) # 3. The parent passes today's contract on the carried bytes. parent_contract = replay_parent_contract( @@ -2272,7 +2488,7 @@ def certify_recertification( ) -> Path: """Write a certified child next to the pending candidate, or raise. - The loader replay runs under the installed runtime with the four wheels; + The loader replay runs under the installed runtime with the three wheels; the three score documents must be the ones the drift recomputes from. The claim is declared here and recorded as the declarer's, with the drift summary beside it. The candidate and its H5 are left unchanged. @@ -2463,20 +2679,33 @@ def require_parent_publication( "to authenticate where the parent's artifacts are published" ) expected = {} + forbidden = set() for name, sha in _mapping(parent.get("release_files")).items(): expected[f"releases/{parent_id}/{name}"] = (sha, release_dir / name) + forbidden.add(name) root = Path(artifact_root) if artifact_root is not None else None for path, sha in _mapping(parent.get("root_artifacts")).items(): expected[path] = (sha, None if root is None else root / path) + if "/" not in path: + forbidden.add(f"releases/{parent_id}/{path}") published = { getattr(info, "path", None): info for info in get_paths_info( repo_id=repo_id, - paths=sorted(expected), + paths=sorted({*expected, *forbidden}), revision=revision, repo_type="dataset", ) } + # One location per artifact: a parent that publishes an artifact at both + # its release-file and its root location leaves the child's placement + # ambiguous, and the publisher would upload only one of them. + doubled = sorted(path for path in forbidden if path in published) + if doubled: + raise ValueError( + f"parent {parent_id!r} publishes {doubled} at both a release-file and " + "a root location; the child's placement of them is ambiguous" + ) for path_in_repo, (sha, local) in sorted(expected.items()): info = published.get(path_in_repo) if info is None: diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index 3e5bba305..3539a4f0a 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -258,6 +258,23 @@ def rename(manifest): recert.describe_parent(release_dir, artifact_root) +@pytest.mark.parametrize( + "path", ["releases/legacy/evidence.json", "./populace_us_2024.h5", "/abs.h5"] +) +def test_describe_parent_refuses_paths_the_publisher_does_not_write(parent, path): + """Sol r2 finding 4: a releases/ root artifact is never uploaded.""" + + release_dir, artifact_root = parent + _rewrite_json( + release_dir / "release_manifest.json", + lambda manifest: manifest["artifacts"]["populace_us_2024_calibration"].update( + path=path + ), + ) + with pytest.raises(ValueError, match="not a release file or root artifact path"): + recert.describe_parent(release_dir, artifact_root) + + def test_describe_parent_refuses_an_artifact_that_differs_from_its_pin(parent): release_dir, artifact_root = parent (artifact_root / "populace_us_2024.h5").write_bytes(b"other bytes") @@ -341,6 +358,32 @@ def test_the_substitution_does_not_excuse_an_unrelated_gate_disagreement( ) +def test_the_substitution_compares_gate_blocks_as_written( + parent, tmp_path, monkeypatch +): + """Sol r2 finding 5: [] and {} disagree, as the evidence check says.""" + + release_dir, artifact_root = parent + _strip_stored_input_evidence(release_dir) + _rewrite_json( + release_dir / "build_manifest.json", + lambda build: build.update(gate_evidence=[]), + ) + _rewrite_json( + release_dir / "release_manifest.json", + lambda manifest: manifest["build"].update(gate_evidence={}), + ) + tool = load_tool() + monkeypatch.setattr(tool, "_producer_identity", lambda: dict(FIXTURE_PRODUCER)) + with pytest.raises(ValueError, match="fails the release contract"): + tool.build_candidate( + parent_release_dir=release_dir, + parent_artifact_root=artifact_root, + output_dir=tmp_path / "out", + release_id=CHILD_ID, + ) + + def test_a_parent_without_stored_input_evidence_is_replaced_not_waived( parent, tmp_path, monkeypatch ): @@ -802,7 +845,24 @@ def reclassify(report): ), ) seed_published_parent(paths_hub, parent_dir, parent_root) - with pytest.raises(ValueError, match="publishes no"): + with pytest.raises(ValueError, match="at both a release-file and a root"): + _publish(paths_hub, output, artifact_root, monkeypatch) + assert paths_hub.events == [] + + +def test_publication_refuses_a_parent_that_publishes_an_artifact_twice( + paths_hub, parent, candidate, tmp_path, monkeypatch +): + """Sol r2 finding 3: a duplicate release-dir copy of a root artifact.""" + + parent_dir, parent_root = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + seed_published_parent(paths_hub, parent_dir, parent_root) + npz = "populace_us_2024_calibration.npz" + paths_hub._commits["commit-parent"][f"releases/{parent_dir.name}/{npz}"] = ( + artifact_root / npz + ).read_bytes() + with pytest.raises(ValueError, match="at both a release-file and a root"): _publish(paths_hub, output, artifact_root, monkeypatch) assert paths_hub.events == [] @@ -898,6 +958,10 @@ def test_preflight_prints_the_recertification_measurement( # --------------------------------------------------------------------------- +import importlib.util # noqa: E402 +from types import SimpleNamespace # noqa: E402 + + def _fake_install(root, name, version): """An installed distribution and its wheel, sharing every package byte.""" @@ -936,7 +1000,6 @@ def fake_runtime(tmp_path, monkeypatch): for name, version in ( ("policyengine-us", "2.60.4"), ("policyengine-core", "3.33.3"), - ("policyengine", "6.10.1"), ("spm-calculator", "1.0.0"), ) } @@ -945,7 +1008,13 @@ def fake_runtime(tmp_path, monkeypatch): "distribution", lambda name: metadata.PathDistribution(installs[name][0]), ) - return tmp_path / "site", tuple(wheel for _, wheel in installs.values()) + site = tmp_path / "site" + monkeypatch.setattr( + importlib.util, + "find_spec", + lambda name: SimpleNamespace(origin=str(site / name / "__init__.py")), + ) + return site, tuple(wheel for _, wheel in installs.values()) def test_runtime_identity_covers_parameters_and_ignores_installer_files(fake_runtime): @@ -977,10 +1046,78 @@ def test_a_patched_parameter_file_is_not_the_wheels_runtime(fake_runtime): assert unbound["policyengine-us"]["wheel_sha256"] is None -def test_the_runtime_needs_all_four_wheels(fake_runtime): +def test_the_runtime_needs_all_three_wheels(fake_runtime): + _, wheels = fake_runtime + with pytest.raises(ValueError, match="all three wheels"): + recert.runtime_identities(wheels[:2], require_wheels=True) + + +def test_an_unrecorded_module_in_the_package_tree_is_refused(fake_runtime): + """Sol r2 finding 1: Core imports every variable module it finds.""" + + site, wheels = fake_runtime + (site / "policyengine_us" / "z_extra.py").write_text("VALUE = 2\n") + with pytest.raises(ValueError, match="never recorded"): + recert.runtime_identities(wheels, require_wheels=True) + + +def test_an_import_that_resolves_elsewhere_is_refused( + fake_runtime, monkeypatch, tmp_path +): _, wheels = fake_runtime - with pytest.raises(ValueError, match="all four wheels"): - recert.runtime_identities(wheels[:3], require_wheels=True) + monkeypatch.setattr( + importlib.util, + "find_spec", + lambda name: SimpleNamespace( + origin=str(tmp_path / "shadow" / name / "__init__.py") + ), + ) + with pytest.raises(ValueError, match="does not resolve to the installed"): + recert.runtime_identities(wheels, require_wheels=True) + + +def test_bytecode_is_isolated_before_the_packages_load(monkeypatch): + """Sol r2 finding 2: a replaced .pyc would run instead of the source.""" + + import sys + import types + + monkeypatch.setattr(recert, "_ISOLATED_PYCACHE", None) + monkeypatch.setattr(sys, "pycache_prefix", None) + monkeypatch.setitem( + sys.modules, "policyengine_us", types.ModuleType("policyengine_us") + ) + with pytest.raises(RuntimeError, match="imported before its bytecode"): + recert.isolate_bytecode() + monkeypatch.delitem(sys.modules, "policyengine_us") + for name in list(sys.modules): + if name.split(".", 1)[0] in recert.IMPORT_NAMES.values(): + monkeypatch.delitem(sys.modules, name) + prefix = recert.isolate_bytecode() + assert sys.pycache_prefix == prefix and not os.listdir(prefix) + assert recert.isolate_bytecode() == prefix + + +def test_a_module_compiled_outside_the_isolated_cache_is_refused( + fake_runtime, monkeypatch, tmp_path +): + import sys + import types + + site, _ = fake_runtime + monkeypatch.setattr(recert, "_ISOLATED_PYCACHE", str(tmp_path / "isolated")) + (tmp_path / "isolated").mkdir() + module = types.ModuleType("policyengine_us.rates") + module.__file__ = str(site / "policyengine_us" / "__init__.py") + module.__cached__ = str(site / "policyengine_us" / "__pycache__" / "x.pyc") + for name in list(sys.modules): + if name.split(".", 1)[0] in recert.IMPORT_NAMES.values(): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "policyengine_us.rates", module) + with pytest.raises(RuntimeError, match="not compiled from the verified"): + recert.verify_loaded_modules() + module.__cached__ = str(tmp_path / "isolated" / "x.pyc") + assert recert.verify_loaded_modules() == 1 def test_the_scorer_refuses_a_nonpositive_batch_size_before_any_work( diff --git a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py index 06ee31e73..3acbcf929 100644 --- a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py +++ b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py @@ -79,6 +79,9 @@ def _write_h5(path): @pytest.fixture def scorable(tmp_path, monkeypatch): + # The scorer compiles the engine into an isolated cache; this module runs + # in its own process, so isolate before anything imports the engine. + recert.isolate_bytecode() artifact_root = tmp_path / "artifacts" artifact_root.mkdir() h5 = artifact_root / "populace_us_2024.h5" diff --git a/test_support/microcosm_data/compatibility_recertification.py b/test_support/microcosm_data/compatibility_recertification.py index fffab75a4..8343358a5 100644 --- a/test_support/microcosm_data/compatibility_recertification.py +++ b/test_support/microcosm_data/compatibility_recertification.py @@ -40,13 +40,11 @@ def _identity(version, digit): TESTED_PACKAGES = { "policyengine-us": _identity("2.60.4", "1"), "policyengine-core": _identity("3.33.3", "3"), - "policyengine": _identity("6.10.1", "5"), "spm-calculator": _identity("1.0.0", "7"), } BASELINE_PACKAGES = { "policyengine-us": _identity("1.729.0", "a"), "policyengine-core": _identity("3.19.0", "c"), - "policyengine": _identity("6.10.1", "5"), "spm-calculator": _identity("1.0.0", "7"), } HOUSEHOLDS = 3 diff --git a/tools/build_us_compatibility_recertification.py b/tools/build_us_compatibility_recertification.py index 8b40e892d..e008e91ca 100644 --- a/tools/build_us_compatibility_recertification.py +++ b/tools/build_us_compatibility_recertification.py @@ -535,7 +535,7 @@ def _pypi_wheel_sha256(name: str, version: str, filename: str) -> str | None: def scoring_runtime(compatibility_wheels: Sequence[Path]) -> dict: - """The four packages' identities, each wheel checked against PyPI's digest. + """The three packages' identities, each wheel checked against PyPI's digest. ``runtime_identities`` proves every installed byte against its wheel; this adds that the wheel is the one PyPI publishes for that version, so a @@ -573,14 +573,19 @@ def score_candidate( ) -> dict: """Score the candidate's H5 under the installed engine; write the document. - The runtime is proven against the four wheels, and they against PyPI, + The runtime is proven against the three wheels, and they against PyPI, before any engine is built, so a long run cannot end on an unprovable - runtime. + runtime. The engine then compiles from those sources into an isolated + bytecode cache, and every module it loaded is checked against the + verified trees before the document is written. """ if type(batch_households) is not int or batch_households <= 0: raise ValueError("--batch-households must be a positive integer") runtime = scoring_runtime(compatibility_wheels) + # Before any engine import: compile the verified sources, never install-tree + # bytecode (Sol r2 finding 2). + recert.isolate_bytecode() import numpy as np from policyengine_us.data import USSingleYearDataset @@ -691,6 +696,7 @@ def score_candidate( "metrics": len(selected), } ) + recert.verify_loaded_modules() document = { "schema_version": recert.SCORES_SCHEMA_VERSION, "kind": recert.SCORES_KIND, @@ -765,7 +771,7 @@ def main(argv: Sequence[str] | None = None) -> int: action="append", type=Path, required=True, - help="the installed country, Core, wrapper and calculator wheels (all four)", + help="the installed country, Core and calculator wheels (all three)", ) args = parser.parse_args(argv) try: From 7f202d7588a2b682dfbcd6939f6491121d051908 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 16:13:40 -0400 Subject: [PATCH 09/12] Address Sol review r3; compare PyTables booleans as booleans - Bytecode trust moves to a bootstrap: the tool sets a fresh sys.pycache_prefix before its first Microcosm import, other entry points start with PYTHONPYCACHEPREFIX, and the replay and scorer refuse a process in which any Microcosm or engine module ran from other bytecode. Score documents must record isolated bytecode. - Runtime trees: only .pyc inside __pycache__ is excluded, so source hidden there or a sourceless .pyc beside the sources is unrecorded and refused; symlinks in a package tree are refused; loaded engine modules are matched by file, covering Core's synthetic variable-module names. - Parent paths with '..' are refused, and the builder checks containment before placing any file. - Docs: three wheels, wrapper status, the bytecode requirement per step. - The loader replay compares a PyTables bool column (stored as uint8) as bool; the real-file rehearsal found the strict dtype comparison refused is_related_to_head_or_spouse though its values were unchanged. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/us-compatibility-recertification.md | 32 ++- .../data/compatibility_recertification.py | 246 ++++++++++++------ .../test_compatibility_recertification.py | 114 +++++--- ...t_compatibility_recertification_scoring.py | 11 +- .../compatibility_recertification.py | 1 + .../build_us_compatibility_recertification.py | 25 +- 7 files changed, 300 insertions(+), 131 deletions(-) diff --git a/README.md b/README.md index cb42f8fb3..32912b3a6 100644 --- a/README.md +++ b/README.md @@ -351,7 +351,7 @@ default. A [compatibility recertification](docs/us-compatibility-recertification.md) republishes a published US calibration release byte for byte under a new id, with a measured `compatible_model_packages` claim for a newer policyengine-us: -a loader replay under the four tested wheels, the engine's reviewed column +a loader replay under the tested country, Core and calculator wheels, the engine's reviewed column changes, and the drift of the critical rows and headline totals between the built-with and tested engines. It never recalibrates. Publication refuses a child whose parent tag on the Hub does not serve the carried manifest bytes. diff --git a/docs/us-compatibility-recertification.md b/docs/us-compatibility-recertification.md index cbb563e0f..019c9bd70 100644 --- a/docs/us-compatibility-recertification.md +++ b/docs/us-compatibility-recertification.md @@ -157,10 +157,22 @@ them. Every score document must: on the path; - each wheel's SHA-256 matches the digest PyPI publishes for that version. -The scorer and the loader replay also compile the packages from source into a -fresh, empty bytecode cache (`sys.pycache_prefix`), so a replaced `.pyc` in the -install tree is never run. After the run they check that every loaded module -came from the verified tree and was compiled into that cache. +CPython runs a cached `.pyc` whose header matches its source without reading +the source, so the proof also covers what actually ran: + +- **Isolated cache.** Every Microcosm and engine module must be compiled from + source into a fresh, empty bytecode cache (`sys.pycache_prefix`), set before + the first Microcosm import. The tool does this at the top of its script. + Every other entry point (`python -m microcosm.data.compatibility_recertification`, + `microcosm-publish-release` / `tools/publish_release.sh`) must be started with + `PYTHONPYCACHEPREFIX` pointing at a new empty directory. The loader replay + and the scorer refuse a process in which any such module ran from other + bytecode. +- **Verified files.** After a run, every module loaded from the engine trees, + matched by file rather than name (Core registers variable modules under + synthetic names), must be a `.py` file of the verified inventory. +- **Trusted startup.** The interpreter, the standard library and site + configuration (`.pth` files, import hooks) are assumed trustworthy. The tested runs must be exactly the runtime the loader replay certified: version, content digest and wheel digest of all three packages. The baseline @@ -215,12 +227,12 @@ recorded commit. 5. **Certify** in the tested environment with the declarer's ruling: ```bash - TESTED/bin/python -m microcosm.data.compatibility_recertification --certify \ + PYTHONPYCACHEPREFIX="$(mktemp -d)" TESTED/bin/python -m microcosm.data.compatibility_recertification --certify \ --release-dir /path/candidate/releases/CHILD_ID \ --artifact-root /path/candidate/artifacts \ --output-dir /path/certified/releases/CHILD_ID \ --compatibility-wheel policyengine_us-X.whl --compatibility-wheel policyengine_core-Y.whl \ - --compatibility-wheel policyengine-Z.whl --compatibility-wheel spm_calculator-W.whl \ + --compatibility-wheel spm_calculator-W.whl \ --baseline-scores baseline.json --tested-scores tested.json \ --counterfactual-scores counterfactual.json \ --compatible-model-specifier 'policyengine-us==X' \ @@ -242,15 +254,15 @@ recorded commit. 6. **Preflight**, offline, and then **publish** on the owner's go: ```bash - microcosm-publish-release /path/certified/releases/CHILD_ID \ + PYTHONPYCACHEPREFIX="$(mktemp -d)" microcosm-publish-release /path/certified/releases/CHILD_ID \ --repo-id policyengine/populace-us \ --artifact-root /path/candidate/artifacts \ - --compatibility-wheel ... (three) \ + --compatibility-wheel ... (the three tested wheels) \ --no-latest --tag-only --preflight-only ``` - Publication is the same command through `tools/publish_release.sh` without - `--preflight-only`. Whether the child should also become `latest.json` is + Publication is the same command through `tools/publish_release.sh`, still + with a fresh `PYTHONPYCACHEPREFIX` and without `--preflight-only`. Whether the child should also become `latest.json` is a separate decision; the commands above leave the pointer alone. 7. **Certify the bundle downstream.** policyengine.py certifies the child on diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py index 9a63e3389..cc4d69dd1 100644 --- a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -22,8 +22,9 @@ **What is measured, and how it is bound.** - *Loader replay* (:func:`run_recertification_loader_replay`). Under the - tested country, Core, wrapper and calculator wheels, both native loaders read - the H5. Every stored column is classified against the tested engine and the + tested country, Core and calculator wheels, the country loader reads the H5 + (the wrapper's loader is the downstream policyengine.py certification's to + prove; :data:`WRAPPER_LOADER_STATUS`). Every stored column is classified against the tested engine and the stored-input register (:mod:`microcosm.data.stored_inputs`). A column the built-with engine read and the tested engine ignores, or one the register says no engine reads and the tested engine now reads, must have an entry in @@ -125,8 +126,10 @@ ) COMPATIBILITY_PACKAGES = tuple(IMPORT_NAMES) #: The wrapper is not part of it. policyengine.py imports its bundled country -#: model at ``import policyengine`` and certifies the bundle's data release -#: against the installed country version there: policyengine 6.10.1 under +#: model with its US package (``import policyengine`` does so unless +#: ``POLICYENGINE_SKIP_COUNTRY_IMPORTS`` is set, and the US dataset loader +#: always does) and certifies the bundle's data release against the installed +#: country version there: policyengine 6.10.1 under #: policyengine-us 2.60.4 raises "Data release manifest is not certified for #: the runtime model version 2.60.4" before any loader exists. A wrapper that #: loads this child is one whose bundle certifies this child, which is the @@ -137,9 +140,10 @@ "package": "policyengine", "reason": ( "policyengine.py certifies its bundled data release against the " - "installed country model at import, so no released wrapper imports " - "under a country version its bundle does not certify; the wrapper " - "loader is proven when a policyengine.py bundle certifies this child" + "installed country model when its US package is imported, so no " + "released wrapper loads US data under a country version its bundle " + "does not certify; the wrapper loader is proven when a policyengine.py " + "bundle certifies this child" ), } #: The producer inventory the release assembler, scorer and certifier run @@ -382,14 +386,20 @@ def write_json(path: Path, value: object) -> None: def _identity_file(name: str) -> bool: """Whether an installed or wheel path belongs to the package's identity. - Bytecode caches and installer bookkeeping are excluded; so are paths the - installer generates outside the package (console scripts, ``../bin``). + Bytecode in ``__pycache__`` and installer bookkeeping are excluded; so are + paths the installer generates outside the package (console scripts, + ``../bin``). Everything else counts, parameter YAML and data files included, because every byte of it can change what the engine computes. """ parts = name.split("/") - if ".." in parts or "__pycache__" in parts or name.endswith(".pyc"): + if ".." in parts: + return False + # Only bytecode CPython writes into a __pycache__ directory is excluded. A + # source file there, or a .pyc beside the sources (which an import loads + # without any source), counts and so must be recorded. + if len(parts) > 1 and parts[-2] == "__pycache__" and parts[-1].endswith(".pyc"): return False return not (parts[0].endswith(".dist-info") and parts[-1] in _INSTALLER_FILES) @@ -404,7 +414,8 @@ def _package_tree_files(dist, identity_files: Sequence[str]) -> set[str]: ``dist.files`` is the installer's record, not the tree: a module dropped into an installed package after installation is absent from it yet importable, and Core loads every variable module it finds. So the tree is - listed from disk and must equal the record (bytecode caches aside). + listed from disk and must equal the record (bytecode caches aside), and + a symlink anywhere in it is refused. """ roots = { @@ -412,15 +423,24 @@ def _package_tree_files(dist, identity_files: Sequence[str]) -> set[str]: for relative in identity_files if "/" in relative and not relative.split("/", 1)[0].endswith(".dist-info") } + import os + found = set() for root in roots: base = Path(dist.locate_file(root)) - for path in base.rglob("*"): - if path.is_dir(): - continue - relative = f"{root}/{path.relative_to(base).as_posix()}" - if _identity_file(relative): - found.add(relative) + for directory, subdirectories, filenames in os.walk(base, followlinks=False): + for name in (*subdirectories, *filenames): + path = Path(directory) / name + if path.is_symlink(): + # Core globs every subdirectory; a link would bring files + # from outside the verified tree into it. + raise ValueError(f"symlink in the installed package tree: {path}") + for name in filenames: + relative = ( + f"{root}/{(Path(directory) / name).relative_to(base).as_posix()}" + ) + if _identity_file(relative): + found.add(relative) return found @@ -445,7 +465,7 @@ def runtime_identities( wheels and refuses an editable install. Nothing here depends on where a wheel was installed from, so honest replays in two environments agree. - This proves the files; :func:`isolate_bytecode` and + This proves the files; :func:`require_trusted_bytecode` and :func:`verify_loaded_modules` prove that what ran was compiled from them. Raises: @@ -547,95 +567,142 @@ def runtime_identities( ) identity["wheel_sha256"] = sha256_file(path) identities[name] = identity + _VERIFIED_TREES[name] = ( + Path(dist.locate_file(IMPORT_NAMES[name])).resolve(), + Path(dist.locate_file("")).resolve(), + frozenset(files), + ) return identities -#: The empty bytecode cache this process compiles the packages into -#: (:func:`isolate_bytecode`); ``None`` until it is created. -_ISOLATED_PYCACHE: str | None = None +#: package -> (import root, site directory, verified files), as the last +#: :func:`runtime_identities` call in this process proved them. +_VERIFIED_TREES: dict[str, tuple[Path, Path, frozenset[str]]] = {} + +#: The environment variable that points CPython's bytecode cache somewhere +#: else at interpreter start (``sys.pycache_prefix``). +PYCACHE_PREFIX_ENVIRONMENT = "PYTHONPYCACHEPREFIX" + + +def bootstrap_bytecode_isolation() -> str: + """Point this process's bytecode cache at a fresh empty directory. + + CPython runs a cached ``.pyc`` whose header matches its source's size and + timestamp without reading the source, so a replaced cache payload runs + code the authenticated source does not contain. A trusted entry point + calls this before it imports any Microcosm module or engine package, so + every module after it compiles from source. An interpreter started with + :data:`PYCACHE_PREFIX_ENVIRONMENT` already has a prefix, which is kept. + Returns the prefix. + """ -def _loaded_package_modules() -> list: import sys - roots = set(IMPORT_NAMES.values()) - return [ - module - for name, module in list(sys.modules.items()) - if name.split(".", 1)[0] in roots and module is not None - ] + if sys.pycache_prefix is None: + sys.pycache_prefix = tempfile.mkdtemp(prefix="recertification-pycache-") + return sys.pycache_prefix -def isolate_bytecode() -> str: - """Compile the packages from source into a fresh, empty bytecode cache. +def _trusted_module(name: str, origin: object) -> bool: + """Whether a loaded module's bytecode must come from the isolated cache.""" - CPython runs a cached ``.pyc`` whose header matches its source's size and - timestamp without reading the source, so a replaced cache payload would - run code the authenticated source does not contain. Pointing - ``sys.pycache_prefix`` at a new empty directory before the packages are - imported means every module they load is compiled from its source. - Call it before importing any of the packages; it refuses if one is - already loaded under another cache. + roots = {"microcosm", "tools", *IMPORT_NAMES.values()} + if name.split(".", 1)[0] in roots: + return True + if not isinstance(origin, str): + return False + resolved = Path(origin).resolve() + return any(resolved.is_relative_to(root) for root, _, _ in _VERIFIED_TREES.values()) - Raises: - RuntimeError: A package module was imported before isolation. + +def bytecode_isolation_status() -> dict: + """Whether every Microcosm and engine module ran from source compiled here. + + Each loaded module of the ``microcosm`` namespace, the ``tools`` package + and the three engine packages (by name, or by file under a verified + package tree, which covers the variable modules Core loads under + synthetic names) must either carry no ``__cached__`` (a script, a + namespace) or have it under ``sys.pycache_prefix``. A module read from + install-tree bytecode, or any module when no prefix is set, is reported. """ import sys - global _ISOLATED_PYCACHE - if _ISOLATED_PYCACHE is not None and sys.pycache_prefix == _ISOLATED_PYCACHE: - return _ISOLATED_PYCACHE - loaded = [module.__name__ for module in _loaded_package_modules()] - if loaded: + prefix = sys.pycache_prefix + offending = [] + for name, module in list(sys.modules.items()): + if module is None: + continue + origin = getattr(module, "__file__", None) + if not _trusted_module(name, origin): + continue + if isinstance(origin, str) and origin.endswith(".pyc"): + offending.append(name) + continue + cached = getattr(module, "__cached__", None) + if cached is None: + continue + if prefix is None or not Path(cached).resolve().is_relative_to( + Path(prefix).resolve() + ): + offending.append(name) + return { + "isolated": prefix is not None and not offending, + "offending": sorted(offending)[:10], + } + + +def require_trusted_bytecode() -> None: + """Refuse unless :func:`bytecode_isolation_status` is isolated. + + Raises: + RuntimeError: Some Microcosm or engine module ran from bytecode the + process did not compile from its source. + """ + + status = bytecode_isolation_status() + if not status["isolated"]: raise RuntimeError( - "the country, Core or calculator package was imported before its " - f"bytecode was isolated ({sorted(loaded)[:3]}); run the replay or " - "scorer in a fresh process" + "Microcosm and engine modules must run from source compiled into a " + "fresh bytecode cache: run through " + "tools/build_us_compatibility_recertification.py, or start Python " + f"with {PYCACHE_PREFIX_ENVIRONMENT} set to a new empty directory " + f"(read from other bytecode: {status['offending']})" ) - _ISOLATED_PYCACHE = tempfile.mkdtemp(prefix="recertification-pycache-") - sys.pycache_prefix = _ISOLATED_PYCACHE - return _ISOLATED_PYCACHE def verify_loaded_modules() -> int: - """Every loaded module of the packages came from their verified trees. + """Every loaded module from the engine trees is a verified source file. - Each must have been read from a file :func:`runtime_identities` covers, - under its installed package, and compiled into the isolated cache. Returns - how many modules were checked. + Matched by file, not name, so the variable modules Core loads under + synthetic names are covered. Each must be a ``.py`` file of the inventory + :func:`runtime_identities` verified. Returns how many were checked. Raises: - RuntimeError: A module was loaded from elsewhere or from install-tree - bytecode. + RuntimeError: No inventory was verified in this process, or a module + came from a file outside it. """ - from importlib import metadata + import sys - if _ISOLATED_PYCACHE is None: - raise RuntimeError("isolate_bytecode() must run before the packages load") - roots = { - import_name: Path( - metadata.distribution(package).locate_file(import_name) - ).resolve() - for package, import_name in IMPORT_NAMES.items() - } - prefix = Path(_ISOLATED_PYCACHE).resolve() + if set(_VERIFIED_TREES) != set(COMPATIBILITY_PACKAGES): + raise RuntimeError("runtime_identities() must verify the packages first") checked = 0 - for module in _loaded_package_modules(): - origin = getattr(module, "__file__", None) - if origin is None: + for name, module in list(sys.modules.items()): + origin = getattr(module, "__file__", None) if module is not None else None + if not isinstance(origin, str): continue - root = roots[module.__name__.split(".", 1)[0]] - cached = getattr(module, "__cached__", None) - if not Path(origin).resolve().is_relative_to(root) or ( - cached is not None and not Path(cached).resolve().is_relative_to(prefix) - ): - raise RuntimeError( - f"{module.__name__} was loaded from {origin} (bytecode {cached}), " - "not compiled from the verified installation" - ) - checked += 1 + path = Path(origin) + for package, (root, site, files) in _VERIFIED_TREES.items(): + if not path.resolve().is_relative_to(root): + continue + relative = path.resolve().relative_to(site).as_posix() + if path.suffix != ".py" or relative not in files: + raise RuntimeError( + f"{name} was loaded from {origin}, not a verified {package} source" + ) + checked += 1 return checked @@ -730,6 +797,7 @@ def describe_parent( path.startswith("/") or "\\" in path or posixpath.normpath(path) != path + or ".." in path.split("/") or path.split("/", 1)[0] == "releases" ): raise ValueError( @@ -909,6 +977,10 @@ def _h5_values(table, column): import numpy as np values = table[column] + # PyTables stores a pandas bool column as a one-byte bitfield, which h5py + # reads as uint8, and records the logical type in ``_dtype``. + if table.attrs.get(f"{column}_dtype") in (b"bool", "bool"): + return np.asarray(values).astype(bool) if values.dtype.kind == "S": return np.asarray([value.decode() for value in values], dtype=object) return values @@ -945,9 +1017,11 @@ def run_recertification_loader_replay( 1. The three packages' installed contents, parameters included, bound to the three wheels when ``require_wheels`` (:func:`runtime_identities`), - compiled from source into an isolated cache (:func:`isolate_bytecode`) - and, after the run, every loaded module from the verified trees - (:func:`verify_loaded_modules`). + run from source compiled into this process's isolated bytecode cache + (:func:`require_trusted_bytecode`, which the caller's trusted entry + point set up with :func:`bootstrap_bytecode_isolation` or + ``PYTHONPYCACHEPREFIX``) and, after the run, every module loaded from + the engine trees a verified source file (:func:`verify_loaded_modules`). 2. The stored-input classification (:func:`stored_column_changes`) and the register digests. An unreviewed change refuses the replay. 3. The country loader (``USSingleYearDataset``) returns every stored @@ -962,7 +1036,7 @@ def run_recertification_loader_replay( step, outside the receipt. """ - isolate_bytecode() + require_trusted_bytecode() import inspect @@ -1047,6 +1121,7 @@ def run_recertification_loader_replay( ) checks.append("country:complete_household_native_role_overrides_default") verify_loaded_modules() + require_trusted_bytecode() checks.append("runtime:every_loaded_module_compiled_from_the_verified_tree") columns = {column for names in tables.values() for column in names} return { @@ -1530,6 +1605,11 @@ def _score_document_failures( f"{label} scores must cover all {expected_households} households of " "the H5 in both their household and their state passes" ) + if _mapping(scores.get("bytecode")).get("isolated") is not True: + failures.append( + f"{label} scores must have run from source compiled into an isolated " + "bytecode cache" + ) packages = _mapping(_mapping(scores.get("runtime")).get("packages")) for package in COMPATIBILITY_PACKAGES: identity = _mapping(packages.get(package)) diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index 3539a4f0a..2ea35c86d 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -259,7 +259,13 @@ def rename(manifest): @pytest.mark.parametrize( - "path", ["releases/legacy/evidence.json", "./populace_us_2024.h5", "/abs.h5"] + "path", + [ + "releases/legacy/evidence.json", + "./populace_us_2024.h5", + "/abs.h5", + "../../escaped.npz", + ], ) def test_describe_parent_refuses_paths_the_publisher_does_not_write(parent, path): """Sol r2 finding 4: a releases/ root artifact is never uploaded.""" @@ -1076,48 +1082,98 @@ def test_an_import_that_resolves_elsewhere_is_refused( recert.runtime_identities(wheels, require_wheels=True) -def test_bytecode_is_isolated_before_the_packages_load(monkeypatch): - """Sol r2 finding 2: a replaced .pyc would run instead of the source.""" +def test_bytecode_isolation_requires_a_prefix_and_modules_compiled_under_it( + monkeypatch, tmp_path +): + """Sol r2/r3: a replaced .pyc would run instead of the authenticated source.""" import sys import types - monkeypatch.setattr(recert, "_ISOLATED_PYCACHE", None) monkeypatch.setattr(sys, "pycache_prefix", None) - monkeypatch.setitem( - sys.modules, "policyengine_us", types.ModuleType("policyengine_us") - ) - with pytest.raises(RuntimeError, match="imported before its bytecode"): - recert.isolate_bytecode() - monkeypatch.delitem(sys.modules, "policyengine_us") + assert recert.bytecode_isolation_status()["isolated"] is False + with pytest.raises(RuntimeError, match="PYTHONPYCACHEPREFIX"): + recert.require_trusted_bytecode() + + prefix = tmp_path / "prefix" + prefix.mkdir() + monkeypatch.setattr(sys, "pycache_prefix", str(prefix)) for name in list(sys.modules): - if name.split(".", 1)[0] in recert.IMPORT_NAMES.values(): + if name.split(".", 1)[0] in { + "microcosm", + "tools", + *recert.IMPORT_NAMES.values(), + }: monkeypatch.delitem(sys.modules, name) - prefix = recert.isolate_bytecode() + module = types.ModuleType("microcosm.data.fake") + module.__file__ = str(tmp_path / "fake.py") + module.__cached__ = str(tmp_path / "__pycache__" / "fake.cpython-313.pyc") + monkeypatch.setitem(sys.modules, "microcosm.data.fake", module) + status = recert.bytecode_isolation_status() + assert status == {"isolated": False, "offending": ["microcosm.data.fake"]} + module.__cached__ = str(prefix / "fake.cpython-313.pyc") + assert recert.bytecode_isolation_status()["isolated"] is True + module.__file__ = str(tmp_path / "fake.pyc") + assert recert.bytecode_isolation_status()["isolated"] is False + + +def test_the_bootstrap_keeps_a_startup_prefix_and_otherwise_makes_one(monkeypatch): + import sys + + monkeypatch.setattr(sys, "pycache_prefix", "/already/set") + assert recert.bootstrap_bytecode_isolation() == "/already/set" + monkeypatch.setattr(sys, "pycache_prefix", None) + prefix = recert.bootstrap_bytecode_isolation() assert sys.pycache_prefix == prefix and not os.listdir(prefix) - assert recert.isolate_bytecode() == prefix -def test_a_module_compiled_outside_the_isolated_cache_is_refused( - fake_runtime, monkeypatch, tmp_path -): +def test_loaded_engine_modules_are_matched_by_file_not_name(fake_runtime, monkeypatch): + """Sol r3 finding 1: Core registers variable modules under synthetic names.""" + import sys import types - site, _ = fake_runtime - monkeypatch.setattr(recert, "_ISOLATED_PYCACHE", str(tmp_path / "isolated")) - (tmp_path / "isolated").mkdir() - module = types.ModuleType("policyengine_us.rates") - module.__file__ = str(site / "policyengine_us" / "__init__.py") - module.__cached__ = str(site / "policyengine_us" / "__pycache__" / "x.pyc") - for name in list(sys.modules): - if name.split(".", 1)[0] in recert.IMPORT_NAMES.values(): - monkeypatch.delitem(sys.modules, name) - monkeypatch.setitem(sys.modules, "policyengine_us.rates", module) - with pytest.raises(RuntimeError, match="not compiled from the verified"): + site, wheels = fake_runtime + recert.runtime_identities(wheels, require_wheels=True) + verified = types.ModuleType("us_system.variables.rate") + verified.__file__ = str(site / "policyengine_us" / "__init__.py") + monkeypatch.setitem(sys.modules, "us_system.variables.rate", verified) + assert recert.verify_loaded_modules() >= 1 + stray = types.ModuleType("us_system.variables.extra") + stray.__file__ = str(site / "policyengine_us" / "extra.py") + monkeypatch.setitem(sys.modules, "us_system.variables.extra", stray) + with pytest.raises(RuntimeError, match="not a verified policyengine-us source"): recert.verify_loaded_modules() - module.__cached__ = str(tmp_path / "isolated" / "x.pyc") - assert recert.verify_loaded_modules() == 1 + + +def test_source_hidden_in_pycache_is_unrecorded(fake_runtime): + """Sol r3 finding 1: Core's directory glob descends into __pycache__/.""" + + site, wheels = fake_runtime + cache = site / "policyengine_us" / "__pycache__" + cache.mkdir() + (cache / "__init__.cpython-313.pyc").write_bytes(b"real bytecode is excluded") + recert.runtime_identities(wheels, require_wheels=True) + (cache / "extra.py").write_text("VALUE = 3\n") + with pytest.raises(ValueError, match="never recorded"): + recert.runtime_identities(wheels, require_wheels=True) + + +def test_a_pyc_beside_the_sources_is_unrecorded(fake_runtime): + site, wheels = fake_runtime + (site / "policyengine_us" / "extra.pyc").write_bytes(b"sourceless module") + with pytest.raises(ValueError, match="never recorded"): + recert.runtime_identities(wheels, require_wheels=True) + + +def test_a_symlink_in_the_package_tree_is_refused(fake_runtime, tmp_path): + site, wheels = fake_runtime + outside = tmp_path / "outside" + outside.mkdir() + (outside / "extra.py").write_text("VALUE = 4\n") + (site / "policyengine_us" / "linked").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symlink in the installed package tree"): + recert.runtime_identities(wheels, require_wheels=True) def test_the_scorer_refuses_a_nonpositive_batch_size_before_any_work( diff --git a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py index 3acbcf929..dd8729777 100644 --- a/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py +++ b/packages/microcosm-data/tests/engine_workflow/us/test_compatibility_recertification_scoring.py @@ -79,9 +79,14 @@ def _write_h5(path): @pytest.fixture def scorable(tmp_path, monkeypatch): - # The scorer compiles the engine into an isolated cache; this module runs - # in its own process, so isolate before anything imports the engine. - recert.isolate_bytecode() + # The tool's script bootstrap isolates bytecode before any Microcosm + # import; a pytest process imported Microcosm long before, so the trust + # checks are stubbed here and tested on synthetic modules engine-free. + monkeypatch.setattr(recert, "require_trusted_bytecode", lambda: None) + monkeypatch.setattr(recert, "verify_loaded_modules", lambda: 0) + monkeypatch.setattr( + recert, "bytecode_isolation_status", lambda: {"isolated": True, "offending": []} + ) artifact_root = tmp_path / "artifacts" artifact_root.mkdir() h5 = artifact_root / "populace_us_2024.h5" diff --git a/test_support/microcosm_data/compatibility_recertification.py b/test_support/microcosm_data/compatibility_recertification.py index 8343358a5..3261fea15 100644 --- a/test_support/microcosm_data/compatibility_recertification.py +++ b/test_support/microcosm_data/compatibility_recertification.py @@ -330,6 +330,7 @@ def score_document( }, "filing_status": {}, "tax_unit_roles": {}, + "bytecode": {"isolated": True}, "code": dict(FIXTURE_PRODUCER), } diff --git a/tools/build_us_compatibility_recertification.py b/tools/build_us_compatibility_recertification.py index e008e91ca..26ca97e87 100644 --- a/tools/build_us_compatibility_recertification.py +++ b/tools/build_us_compatibility_recertification.py @@ -37,8 +37,14 @@ from importlib import metadata from pathlib import Path -from microcosm.data import compatibility_recertification as recert -from microcosm.data import contract, source_enrichment, stored_inputs +# Before any Microcosm import, so every Microcosm and engine module this +# process runs compiles from source into a fresh, empty bytecode cache +# (compatibility_recertification.bootstrap_bytecode_isolation; Sol r3). +if __name__ == "__main__" and sys.pycache_prefix is None: + sys.pycache_prefix = tempfile.mkdtemp(prefix="recertification-pycache-") + +from microcosm.data import compatibility_recertification as recert # noqa: E402 +from microcosm.data import contract, source_enrichment, stored_inputs # noqa: E402 _PRODUCER_FILES = recert.PRODUCER_SOURCE_FILES _ROOT = Path(__file__).resolve().parents[1] @@ -209,6 +215,11 @@ def build_candidate( artifact_root.mkdir(mode=0o700) for path in parent["root_artifacts"]: destination = artifact_root / path + # describe_parent refuses '..'; containment is checked again here, + # before anything is created, so no path can place a file outside + # the staging directory (Sol r3 finding 3). + if not destination.resolve().is_relative_to(artifact_root.resolve()): + raise ValueError(f"root artifact {path!r} escapes the artifact root") destination.parent.mkdir(parents=True, exist_ok=True) _place(parent_artifact_root / path, destination) for name in parent["release_files"]: @@ -583,9 +594,9 @@ def score_candidate( if type(batch_households) is not int or batch_households <= 0: raise ValueError("--batch-households must be a positive integer") runtime = scoring_runtime(compatibility_wheels) - # Before any engine import: compile the verified sources, never install-tree - # bytecode (Sol r2 finding 2). - recert.isolate_bytecode() + # Before any engine import: Microcosm's own modules must already have run + # from source compiled here (the bootstrap above), and the engine will too. + recert.require_trusted_bytecode() import numpy as np from policyengine_us.data import USSingleYearDataset @@ -697,6 +708,9 @@ def score_candidate( } ) recert.verify_loaded_modules() + bytecode = recert.bytecode_isolation_status() + if not bytecode["isolated"]: + raise RuntimeError(f"modules ran from untrusted bytecode: {bytecode}") document = { "schema_version": recert.SCORES_SCHEMA_VERSION, "kind": recert.SCORES_KIND, @@ -732,6 +746,7 @@ def score_candidate( "metrics": {metric_id: {"value": value} for metric_id, value in totals.items()}, "filing_status": filing_status, "tax_unit_roles": roles, + "bytecode": {"isolated": True}, "code": producer, } if not all(np.isfinite(value) for value in totals.values()): From 9ee1385fc601dd768f42803572c64c78a9c7ce95 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 17:20:25 -0400 Subject: [PATCH 10/12] Address Sol review r4: overlays, authenticate before import, contain reconstruction - An import must resolve to the installed package as written: the spec's origin and submodule search locations may not pass through a symlink or lie elsewhere, and every loaded module of the three engine packages is checked by name (file and __path__ under its tree, unlinked) as well as by file. - The replay authenticates the runtime before its first engine import, and a process that imported the engine before any authentication is refused, since Core executes every variable source when the system is built. - Parent reconstruction validates the parent id and carried file names and contains every write in staging; the validator stops at a parent block the carried bytes do not describe. Co-Authored-By: Claude Opus 5.5 --- .../data/compatibility_recertification.py | 145 ++++++++++++++++-- .../test_compatibility_recertification.py | 105 ++++++++++++- .../build_us_compatibility_recertification.py | 4 +- 3 files changed, 234 insertions(+), 20 deletions(-) diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py index cc4d69dd1..ffa550ac9 100644 --- a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -58,6 +58,7 @@ import hashlib import json import math +import os import posixpath import re import shutil @@ -531,10 +532,16 @@ def runtime_identities( ) spec = importlib.util.find_spec(IMPORT_NAMES[name]) installed_root = Path(dist.locate_file(IMPORT_NAMES[name])).resolve() + # Compared as written, not resolved: an overlay package earlier on the + # path whose __init__ is a symlink to the installed one resolves to the + # installed file, while Python searches the overlay for submodules. + search = list(getattr(spec, "submodule_search_locations", None) or ()) if ( spec is None or spec.origin is None - or not Path(spec.origin).resolve().is_relative_to(installed_root) + or not _unlinked_under(spec.origin, installed_root) + or [str(Path(location).absolute()) for location in search] + != [str(installed_root)] ): raise ValueError( f"import {IMPORT_NAMES[name]} does not resolve to the installed " @@ -672,33 +679,98 @@ def require_trusted_bytecode() -> None: ) -def verify_loaded_modules() -> int: - """Every loaded module from the engine trees is a verified source file. +def authenticate_runtime_before_import( + compatibility_wheels: Sequence[Path | str] = (), *, require_wheels: bool +) -> dict: + """:func:`runtime_identities`, refusing a process that ran the engine first. + + Core executes every variable source in the country tree when the system + is built, so a file added to the tree runs on the first engine import. + Authenticating afterwards would check a tree the added code could already + have cleaned up. An engine module loaded before any authentication in this + process is therefore refused; one loaded after it is checked again by + :func:`verify_loaded_modules`. + + Raises: + RuntimeError: An engine package was imported before its runtime was + authenticated in this process. + ValueError: As :func:`runtime_identities`. + """ + + import sys + + roots = set(IMPORT_NAMES.values()) + early = sorted( + name + for name, module in list(sys.modules.items()) + if module is not None and name.split(".", 1)[0] in roots + ) + if early and set(_VERIFIED_TREES) != set(COMPATIBILITY_PACKAGES): + raise RuntimeError( + "the engine was imported before its runtime was authenticated " + f"({early[:3]}); run the replay or scorer in a fresh process" + ) + return runtime_identities(compatibility_wheels, require_wheels=require_wheels) + + +def _unlinked_under(path: str | Path, root: Path) -> bool: + """Whether ``path`` lies under ``root`` as written, with no symlink on the way.""" - Matched by file, not name, so the variable modules Core loads under - synthetic names are covered. Each must be a ``.py`` file of the inventory + written = Path(os.path.abspath(path)) + return written == Path(os.path.realpath(path)) and written.is_relative_to(root) + + +def verify_loaded_modules() -> int: + """Every loaded engine module is a verified source file, by name and by file. + + By file: a module whose file lies under a verified tree, whatever its name, + which covers the variable modules Core loads under synthetic names. By + name: every module of the three engine packages, which must lie under its + package's tree as written (a symlinked overlay earlier on the path + resolves into the tree while Python loads its other files), as must each + package's ``__path__``. Each file must be a ``.py`` file of the inventory :func:`runtime_identities` verified. Returns how many were checked. Raises: RuntimeError: No inventory was verified in this process, or a module - came from a file outside it. + came from elsewhere. """ import sys if set(_VERIFIED_TREES) != set(COMPATIBILITY_PACKAGES): raise RuntimeError("runtime_identities() must verify the packages first") + roots_by_import = { + IMPORT_NAMES[package]: (package, tree) + for package, tree in _VERIFIED_TREES.items() + } checked = 0 for name, module in list(sys.modules.items()): - origin = getattr(module, "__file__", None) if module is not None else None + if module is None: + continue + origin = getattr(module, "__file__", None) + named = roots_by_import.get(name.split(".", 1)[0]) + if named is not None: + package, (root, _, _) = named + locations = list(getattr(module, "__path__", None) or ()) + if ( + not isinstance(origin, str) + or not _unlinked_under(origin, root) + or any(not _unlinked_under(location, root) for location in locations) + ): + raise RuntimeError( + f"{name} was loaded from {origin} (search path {locations}), " + f"not from the installed {package}" + ) if not isinstance(origin, str): continue - path = Path(origin) for package, (root, site, files) in _VERIFIED_TREES.items(): - if not path.resolve().is_relative_to(root): + if not Path(origin).resolve().is_relative_to(root): continue - relative = path.resolve().relative_to(site).as_posix() - if path.suffix != ".py" or relative not in files: + if not _unlinked_under(origin, root): + raise RuntimeError(f"{name} was loaded through a symlink: {origin}") + relative = Path(os.path.abspath(origin)).relative_to(site).as_posix() + if Path(origin).suffix != ".py" or relative not in files: raise RuntimeError( f"{name} was loaded from {origin}, not a verified {package} source" ) @@ -856,10 +928,39 @@ def describe_parent( } +_PARENT_ID = re.compile(r"populace-us-[A-Za-z0-9][A-Za-z0-9._-]*") + + +def _bare_release_filename(name: object) -> bool: + return ( + isinstance(name, str) + and name not in {"", ".", ".."} + and Path(name).name == name + and "\\" not in name + ) + + def _reconstructed_parent(release_dir: Path, parent: Mapping, staging: Path) -> Path: - """The parent's release directory, rebuilt from the bytes the child carries.""" + """The parent's release directory, rebuilt from the bytes the child carries. + + The parent id and every carried file name are checked before anything is + created, and every write is contained in ``staging`` (Sol r4 finding 3). + + Raises: + ValueError: The report names a parent id or file that would leave + the staging directory. + """ - directory = staging / str(parent["build_id"]) + parent_id = parent.get("build_id") + names = list(_mapping(parent.get("release_files"))) + if not isinstance(parent_id, str) or _PARENT_ID.fullmatch(parent_id) is None: + raise ValueError(f"parent build_id {parent_id!r} is not a US release id") + if not all(_bare_release_filename(name) for name in names): + raise ValueError("parent release_files must be bare file names") + staging = Path(staging).resolve() + directory = staging / parent_id + if directory.parent != staging: + raise ValueError(f"parent build_id {parent_id!r} escapes the staging directory") directory.mkdir() shutil.copyfile( release_dir / PARENT_RELEASE_MANIFEST_FILE, directory / "release_manifest.json" @@ -1037,6 +1138,12 @@ def run_recertification_loader_replay( """ require_trusted_bytecode() + # Before the first engine import: importing policyengine_us builds the + # country system, and Core executes every variable source it finds, so + # the tree must be authenticated before anything in it runs (Sol r4). + packages = authenticate_runtime_before_import( + compatibility_wheels, require_wheels=require_wheels + ) import inspect @@ -1048,7 +1155,6 @@ def run_recertification_loader_replay( from microcosm.data import source_enrichment, stored_inputs candidate_h5 = Path(candidate_h5) - packages = runtime_identities(compatibility_wheels, require_wheels=require_wheels) tables = stored_inputs.h5_stored_tables(candidate_h5) engine = stored_inputs.installed_us_engine() changes = stored_column_changes( @@ -2199,9 +2305,14 @@ def validate_recertification_candidate( [f"carried parent does not describe a supported parent: {exc}"], ) from exc if described != dict(parent): - failures.append( - f"{REPORT_FILE} parent block must equal what the carried parent manifest " - "and files describe" + # Nothing later may act on a parent block the carried bytes do not + # describe: it names the files and paths the checks below touch. + raise ReleaseContractError( + release_dir, + [ + f"{REPORT_FILE} parent block must equal what the carried parent " + "manifest and files describe" + ], ) parent_manifest = json.loads(carried_manifest.read_text()) diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index 2ea35c86d..b2e44f119 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -188,6 +188,29 @@ def test_tampered_candidates_are_refused(candidate, tamper, message): ) +@pytest.mark.parametrize( + "build_id", ["ABSOLUTE", "../outside", "populace-us-x/../../y"] +) +def test_parent_reconstruction_never_writes_outside_staging( + candidate, tmp_path, build_id +): + """Sol r4 finding 3: the report's parent id names the rebuild directory.""" + + release_dir, artifact_root = candidate + target = tmp_path / "recert-outside" + bad = str(target) if build_id == "ABSOLUTE" else build_id + _rewrite_json( + release_dir / recert.REPORT_FILE, + lambda report: report["parent"].update(build_id=bad), + ) + with pytest.raises(ReleaseContractError): + recert.validate_recertification_candidate( + release_dir, artifact_root=artifact_root + ) + assert not target.exists() + assert not (release_dir.parent.parent / "outside").exists() + + def test_the_h5_may_not_live_in_the_release_directory(candidate): release_dir, artifact_root = candidate shutil.copyfile( @@ -1018,11 +1041,26 @@ def fake_runtime(tmp_path, monkeypatch): monkeypatch.setattr( importlib.util, "find_spec", - lambda name: SimpleNamespace(origin=str(site / name / "__init__.py")), + lambda name: SimpleNamespace( + origin=str(site / name / "__init__.py"), + submodule_search_locations=[str(site / name)], + ), ) + monkeypatch.setattr(recert, "_VERIFIED_TREES", {}) + _forget_engine_modules(monkeypatch) return site, tuple(wheel for _, wheel in installs.values()) +def _forget_engine_modules(monkeypatch): + """Hide any real engine modules this test process imported earlier.""" + + import sys + + for name in list(sys.modules): + if name.split(".", 1)[0] in recert.IMPORT_NAMES.values(): + monkeypatch.delitem(sys.modules, name) + + def test_runtime_identity_covers_parameters_and_ignores_installer_files(fake_runtime): site, wheels = fake_runtime identities = recert.runtime_identities(wheels, require_wheels=True) @@ -1067,6 +1105,68 @@ def test_an_unrecorded_module_in_the_package_tree_is_refused(fake_runtime): recert.runtime_identities(wheels, require_wheels=True) +def test_a_symlinked_overlay_package_is_refused(fake_runtime, monkeypatch, tmp_path): + """Sol r4 finding 1: the overlay's __init__ resolves into the real tree.""" + + site, wheels = fake_runtime + overlay = tmp_path / "overlay" / "policyengine_core" + overlay.mkdir(parents=True) + (overlay / "__init__.py").symlink_to(site / "policyengine_core" / "__init__.py") + + def find_spec(name): + if name == "policyengine_core": + return SimpleNamespace( + origin=str(overlay / "__init__.py"), + submodule_search_locations=[str(overlay)], + ) + return SimpleNamespace( + origin=str(site / name / "__init__.py"), + submodule_search_locations=[str(site / name)], + ) + + monkeypatch.setattr(importlib.util, "find_spec", find_spec) + with pytest.raises(ValueError, match="does not resolve to the installed"): + recert.runtime_identities(wheels, require_wheels=True) + + +def test_a_loaded_engine_module_outside_its_tree_is_refused_by_name( + fake_runtime, monkeypatch, tmp_path +): + import sys + import types + + site, wheels = fake_runtime + recert.runtime_identities(wheels, require_wheels=True) + package = types.ModuleType("policyengine_core") + package.__file__ = str(site / "policyengine_core" / "__init__.py") + package.__path__ = [str(tmp_path / "overlay" / "policyengine_core")] + monkeypatch.setitem(sys.modules, "policyengine_core", package) + with pytest.raises(RuntimeError, match="not from the installed policyengine-core"): + recert.verify_loaded_modules() + package.__path__ = [str(site / "policyengine_core")] + helper = types.ModuleType("policyengine_core.simulations.microsimulation") + helper.__file__ = str(tmp_path / "overlay" / "microsimulation.py") + monkeypatch.setitem(sys.modules, helper.__name__, helper) + with pytest.raises(RuntimeError, match="not from the installed policyengine-core"): + recert.verify_loaded_modules() + + +def test_the_engine_must_not_run_before_its_runtime_is_authenticated( + fake_runtime, monkeypatch +): + """Sol r4 finding 2: Core runs every variable source on first import.""" + + import sys + import types + + _, wheels = fake_runtime + monkeypatch.setitem( + sys.modules, "policyengine_us", types.ModuleType("policyengine_us") + ) + with pytest.raises(RuntimeError, match="imported before its runtime"): + recert.authenticate_runtime_before_import(wheels, require_wheels=True) + + def test_an_import_that_resolves_elsewhere_is_refused( fake_runtime, monkeypatch, tmp_path ): @@ -1075,7 +1175,8 @@ def test_an_import_that_resolves_elsewhere_is_refused( importlib.util, "find_spec", lambda name: SimpleNamespace( - origin=str(tmp_path / "shadow" / name / "__init__.py") + origin=str(tmp_path / "shadow" / name / "__init__.py"), + submodule_search_locations=[str(tmp_path / "shadow" / name)], ), ) with pytest.raises(ValueError, match="does not resolve to the installed"): diff --git a/tools/build_us_compatibility_recertification.py b/tools/build_us_compatibility_recertification.py index 26ca97e87..23c1ab2e0 100644 --- a/tools/build_us_compatibility_recertification.py +++ b/tools/build_us_compatibility_recertification.py @@ -553,7 +553,9 @@ def scoring_runtime(compatibility_wheels: Sequence[Path]) -> dict: score cannot rest on a locally patched engine bearing a released version. """ - identities = recert.runtime_identities(compatibility_wheels, require_wheels=True) + identities = recert.authenticate_runtime_before_import( + compatibility_wheels, require_wheels=True + ) by_package = {} for wheel in compatibility_wheels: path = Path(wheel) From 142b3cb1f57f21cb7eecdb4d766da1a4fddf0324 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 18:00:37 -0400 Subject: [PATCH 11/12] Refuse a bad fetch-parent release id before creating anything Sol r5 AGREE notes: fetch-parent now validates the US release-id grammar before it makes any directory, and the runbook describes the loaded-module check by package name and by file origin, including search paths. Co-Authored-By: Claude Opus 5.5 --- docs/us-compatibility-recertification.md | 12 +++++++++--- .../shared/test_compatibility_recertification.py | 8 ++++++++ tools/build_us_compatibility_recertification.py | 4 ++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/us-compatibility-recertification.md b/docs/us-compatibility-recertification.md index 019c9bd70..d3f5d5803 100644 --- a/docs/us-compatibility-recertification.md +++ b/docs/us-compatibility-recertification.md @@ -168,9 +168,15 @@ the source, so the proof also covers what actually ran: `PYTHONPYCACHEPREFIX` pointing at a new empty directory. The loader replay and the scorer refuse a process in which any such module ran from other bytecode. -- **Verified files.** After a run, every module loaded from the engine trees, - matched by file rather than name (Core registers variable modules under - synthetic names), must be a `.py` file of the verified inventory. +- **Verified files.** After a run, every loaded engine module is checked + twice. By file: any module whose file lies in an engine tree, whatever its + name (Core registers variable modules under synthetic names), must be a + `.py` file of the verified inventory. By name: every module of the three + engine packages, and each package's search path, must lie inside its + installed tree as written, with no symlink on the way, so an overlay + package earlier on the path is refused. The runtime is authenticated before + the first engine import, because Core executes every variable source when + the country system is built. - **Trusted startup.** The interpreter, the standard library and site configuration (`.pth` files, import hooks) are assumed trustworthy. diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index b2e44f119..b17dad019 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -1292,3 +1292,11 @@ def test_the_scorer_refuses_a_nonpositive_batch_size_before_any_work( compatibility_wheels=(), batch_households=size, ) + + +@pytest.mark.parametrize("release_id", ["/tmp/outside", "../outside", "not-us"]) +def test_fetch_parent_refuses_an_id_before_creating_anything(tmp_path, release_id): + tool = load_tool() + with pytest.raises(ValueError, match="is not a US release id"): + tool.fetch_parent(release_id=release_id, output_dir=tmp_path / "parent") + assert not (tmp_path / "parent").exists() diff --git a/tools/build_us_compatibility_recertification.py b/tools/build_us_compatibility_recertification.py index 23c1ab2e0..7632bcd33 100644 --- a/tools/build_us_compatibility_recertification.py +++ b/tools/build_us_compatibility_recertification.py @@ -110,6 +110,10 @@ def fetch_parent( from huggingface_hub import HfApi, hf_hub_download + # Validated before anything is created, so an id cannot place directories + # outside output_dir (Sol r5 note). + if recert._PARENT_ID.fullmatch(release_id) is None: + raise ValueError(f"{release_id!r} is not a US release id") if output_dir.exists(): raise FileExistsError(f"{output_dir} exists; choose a new directory") release_dir = output_dir / "releases" / release_id From 446b3fe3f2ec2096a6c0aa903dbb4788d47f0adc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 18:19:26 -0400 Subject: [PATCH 12/12] Accept namespace packages by path; keep recertifications off latest.json A real-runtime smoke of the trust checks (both engines, unstubbed) found that verify_loaded_modules refused every namespace package: policyengine-us imports parameter and reform directories without __init__.py, which Python gives no file and a NamespaceLoader. Such a module runs no code, so it is now checked by its search path alone, which must lie inside the installed tree as written. A fileless module without a namespace loader is still refused. The smoke on the fixed code passes under 2.2.1 and 2.60.4, and the loader replay passes unstubbed under 2.60.4. The publisher now refuses latest.json for a recertification child, as it does for the reported-receipt child (owner review, nonblocking note): it republishes its parent's data under a wider claim and is reached by its release id. Co-Authored-By: Claude Opus 5.5 --- docs/us-compatibility-recertification.md | 6 +- .../data/compatibility_recertification.py | 34 +++++++-- .../src/microcosm/data/release.py | 16 +++- .../test_compatibility_recertification.py | 75 +++++++++++++++++++ 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/docs/us-compatibility-recertification.md b/docs/us-compatibility-recertification.md index d3f5d5803..c9cb1cc10 100644 --- a/docs/us-compatibility-recertification.md +++ b/docs/us-compatibility-recertification.md @@ -268,8 +268,10 @@ recorded commit. ``` Publication is the same command through `tools/publish_release.sh`, still - with a fresh `PYTHONPYCACHEPREFIX` and without `--preflight-only`. Whether the child should also become `latest.json` is - a separate decision; the commands above leave the pointer alone. + with a fresh `PYTHONPYCACHEPREFIX` and without `--preflight-only`. The + publisher refuses to point `latest.json` at a recertification child, which + republishes its parent's data under a wider claim: consumers reach it by + its release id, so it always publishes with `--no-latest --tag-only`. 7. **Certify the bundle downstream.** policyengine.py certifies the child on its publisher claim (`compatibility_basis: legacy_compatible_model_package`, diff --git a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py index ffa550ac9..f35d6e6ac 100644 --- a/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py +++ b/packages/microcosm-data/src/microcosm/data/compatibility_recertification.py @@ -720,6 +720,27 @@ def _unlinked_under(path: str | Path, root: Path) -> bool: return written == Path(os.path.realpath(path)) and written.is_relative_to(root) +def _is_namespace_package(module) -> bool: + """Whether ``module`` is a namespace package: a directory, no code run. + + policyengine-us imports parameter and reform directories that have no + ``__init__.py`` (``policyengine_us.parameters.gov``, for one). Python gives + them no file and a namespace loader, which executes nothing, so their + search path is all there is to check. + """ + + from importlib.machinery import NamespaceLoader + + spec = getattr(module, "__spec__", None) + return ( + getattr(module, "__file__", None) is None + and spec is not None + and spec.origin is None + and isinstance(spec.loader, NamespaceLoader) + and bool(list(getattr(module, "__path__", None) or ())) + ) + + def verify_loaded_modules() -> int: """Every loaded engine module is a verified source file, by name and by file. @@ -728,7 +749,8 @@ def verify_loaded_modules() -> int: name: every module of the three engine packages, which must lie under its package's tree as written (a symlinked overlay earlier on the path resolves into the tree while Python loads its other files), as must each - package's ``__path__``. Each file must be a ``.py`` file of the inventory + package's ``__path__``; a namespace package, which has no file, is checked + by its path alone. Each file must be a ``.py`` file of the inventory :func:`runtime_identities` verified. Returns how many were checked. Raises: @@ -753,10 +775,12 @@ def verify_loaded_modules() -> int: if named is not None: package, (root, _, _) = named locations = list(getattr(module, "__path__", None) or ()) - if ( - not isinstance(origin, str) - or not _unlinked_under(origin, root) - or any(not _unlinked_under(location, root) for location in locations) + if isinstance(origin, str): + located = _unlinked_under(origin, root) + else: + located = _is_namespace_package(module) + if not located or any( + not _unlinked_under(location, root) for location in locations ): raise RuntimeError( f"{name} was loaded from {origin} (search path {locations}), " diff --git a/packages/microcosm-data/src/microcosm/data/release.py b/packages/microcosm-data/src/microcosm/data/release.py index d336e320a..80a7a7ed8 100644 --- a/packages/microcosm-data/src/microcosm/data/release.py +++ b/packages/microcosm-data/src/microcosm/data/release.py @@ -314,8 +314,8 @@ def prepare_release( release_type = ( manifest.get("release_type") if isinstance(manifest, Mapping) else None ) - # A recertification replays its loader under the four wheels too, but - # its H5 is its parent's, so it takes no parent_h5. + # A recertification replays its loader under its tested wheels too, + # but its H5 is its parent's, so it takes no parent_h5. if not ( release_type == SOURCE_ENRICHMENT_RELEASE_TYPE or (release_type == RECERTIFICATION_RELEASE_TYPE and parent_h5 is None) @@ -389,6 +389,18 @@ def prepare_release( "pointer. Pass update_latest=False (publish CLI: --no-latest, " "with --tag-only to leave main untouched)." ) + from microcosm.data.compatibility_recertification import is_recertification + + if is_recertification(release_dir): + # A recertification republishes its parent's data under a wider + # compatibility claim, so it never moves the default pointer as a + # side effect; consumers reach it by its release id. + raise ValueError( + f"release {release_id!r} is a compatibility recertification of " + "a published release, never the national default pointer. " + "Pass update_latest=False (publish CLI: --no-latest, with " + "--tag-only to leave main untouched)." + ) if tag_only and update_latest: raise ValueError( "tag_only=True requires update_latest=False; a tag-only publication " diff --git a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py index b17dad019..3f9d9033e 100644 --- a/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py +++ b/packages/microcosm-data/tests/engine_free/shared/test_compatibility_recertification.py @@ -951,6 +951,34 @@ def test_prepare_release_takes_wheels_but_not_a_parent_h5( assert "populace_us_2024.h5" in prepared.root_artifacts +def test_a_recertification_never_moves_latest( + paths_hub, parent, candidate, tmp_path, monkeypatch +): + parent_dir, parent_root = parent + output, artifact_root = certify(candidate, tmp_path, monkeypatch) + seed_published_parent(paths_hub, parent_dir, parent_root) + monkeypatch.setattr(recert, "run_recertification_loader_replay", fake_replay()) + monkeypatch.setattr(recert, "_check_producer", lambda code, failures, label: None) + wheel = tmp_path / "policyengine_us-2.60.4-py3-none-any.whl" + wheel.write_bytes(b"") + with pytest.raises(ValueError, match="never the national default pointer"): + release_module.prepare_release( + output, artifact_root=artifact_root, compatibility_wheels=(wheel,) + ) + events = list(paths_hub.events) + with pytest.raises(ValueError, match="never the national default pointer"): + release_module.publish_release( + output, + "policyengine/populace-us", + api=paths_hub, + artifact_root=artifact_root, + compatibility_wheels=(wheel,), + notify=False, + ) + assert paths_hub.events == events + assert "latest.json" not in dict(paths_hub.uploads) + + def test_preflight_prints_the_recertification_measurement( candidate, tmp_path, monkeypatch, capsys ): @@ -1151,6 +1179,53 @@ def test_a_loaded_engine_module_outside_its_tree_is_refused_by_name( recert.verify_loaded_modules() +def _namespace_module(name, location): + """A module as Python imports a directory without ``__init__.py``.""" + + from importlib.machinery import ModuleSpec, NamespaceLoader, PathFinder + + loader = NamespaceLoader(name, [str(location)], PathFinder._get_spec) + spec = ModuleSpec(name, loader, is_package=True) + spec.submodule_search_locations = [str(location)] + return importlib.util.module_from_spec(spec) + + +def test_a_namespace_package_is_checked_by_its_path( + fake_runtime, monkeypatch, tmp_path +): + """The real-runtime smoke: policyengine_us.parameters.gov has no file.""" + + import sys + import types + + site, wheels = fake_runtime + recert.runtime_identities(wheels, require_wheels=True) + package = types.ModuleType("policyengine_us") + package.__file__ = str(site / "policyengine_us" / "__init__.py") + package.__path__ = [str(site / "policyengine_us")] + monkeypatch.setitem(sys.modules, "policyengine_us", package) + inside = _namespace_module( + "policyengine_us.parameters", site / "policyengine_us" / "parameters" + ) + assert getattr(inside, "__file__", None) is None + assert recert._is_namespace_package(inside) + monkeypatch.setitem(sys.modules, inside.__name__, inside) + assert recert.verify_loaded_modules() >= 1 + outside = _namespace_module( + "policyengine_us.reforms", tmp_path / "overlay" / "reforms" + ) + monkeypatch.setitem(sys.modules, outside.__name__, outside) + with pytest.raises(RuntimeError, match="not from the installed policyengine-us"): + recert.verify_loaded_modules() + monkeypatch.delitem(sys.modules, outside.__name__) + # No file and no namespace loader: code that came from somewhere unchecked. + in_memory = types.ModuleType("policyengine_us.generated") + in_memory.__path__ = [str(site / "policyengine_us")] + monkeypatch.setitem(sys.modules, in_memory.__name__, in_memory) + with pytest.raises(RuntimeError, match="not from the installed policyengine-us"): + recert.verify_loaded_modules() + + def test_the_engine_must_not_run_before_its_runtime_is_authenticated( fake_runtime, monkeypatch ):