From 509e4da6e9ffcff90d33d5c076006a23d7873d18 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:46:50 +0400 Subject: [PATCH 1/4] Generate a Draft 2020-12 JSON Schema for calibration diagnostics schema 8 The collector should check calibration_diagnostics.json at intake without importing Microcosm's pydantic models. The existing calibration_diagnostics_json_schema() returns the parse adapter's union with the legacy schema-6/7 envelope. It declares no dialect and doesn't say which model rules it leaves out. calibration_diagnostics_v8_json_schema() generates the schema from CalibrationDiagnosticsV8 in validation mode and adds the Draft 2020-12 dialect, an $id, a title and a description. The description lists every rule the models enforce that a generated schema cannot express, from the JsonSchemaGap register: cross-field reconciliations, finite numbers, non-blank text, strict integers and digest anchoring under Python's re. It also notes the one place the schema is stricter, literal values, which the models compare with Python equality. The tests pin four things: - every validator function in the schema-8 model tree is listed; - each listed rule has a document the schema accepts and the models refuse; - valid documents are accepted by both; - representative invalid documents are refused by both. The two schema-8 document builders move from test_schema.py to test_support, so both test modules share them. jsonschema, already locked through microcosm-build, joins the microcosm-diagnostics dev group. The primary-QRF worker identity's approved uv.lock digest moves with the lock, as it does for every lock change. Co-Authored-By: Claude Opus 5.5 --- ...diagnostics-json-schema-generator.added.md | 3 + .../build/us_runtime/worker_identity.py | 2 +- packages/microcosm-diagnostics/README.md | 10 + packages/microcosm-diagnostics/pyproject.toml | 4 +- .../src/microcosm/diagnostics/__init__.py | 16 + .../src/microcosm/diagnostics/json_schema.py | 281 ++++++++ .../engine_free/shared/test_json_schema.py | 619 ++++++++++++++++++ .../tests/engine_free/shared/test_schema.py | 258 +------- .../microcosm_diagnostics/__init__.py | 1 + .../microcosm_diagnostics/json_schema.py | 38 ++ .../microcosm_diagnostics/payloads.py | 223 +++++++ uv.lock | 6 +- 12 files changed, 1227 insertions(+), 234 deletions(-) create mode 100644 changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md create mode 100644 packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py create mode 100644 packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py create mode 100644 test_support/microcosm_diagnostics/__init__.py create mode 100644 test_support/microcosm_diagnostics/json_schema.py create mode 100644 test_support/microcosm_diagnostics/payloads.py diff --git a/changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md b/changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md new file mode 100644 index 0000000000..6db55edf79 --- /dev/null +++ b/changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md @@ -0,0 +1,3 @@ +`microcosm.diagnostics.calibration_diagnostics_v8_json_schema()` generates a Draft 2020-12 JSON Schema for calibration diagnostics schema 8 from the pydantic models, so consumers such as the collector can check `calibration_diagnostics.json` without importing them (microcosm#1293). Its description lists every rule the models enforce that the schema cannot express: cross-field reconciliations, finite numbers, non-blank text, strict integers and digest anchoring under Python's `re`. It also notes the one case where the schema is stricter than the models: literal values. Tests pin that every model validator is listed, that each listed rule has a document the schema accepts and the models refuse, and that valid and representative invalid documents are classified the same way by both. + +`jsonschema`, already locked as a runtime dependency of `microcosm-build`, joins the `microcosm-diagnostics` dev group, so no new package enters the lock. The primary-QRF worker identity's approved `uv.lock` digest moves with the lock, as it does for every lock change. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index fb9c1522b7..9b7b2f4e01 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "e5d5818bdc73fd44fbb474b6f0bca4b697ee2f70a0cbb9e47eb04b20f7408d77" + "751721b231ab8cc6b3d10164f61ca429f5171661951494503fb0873e3f67b117" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-diagnostics/README.md b/packages/microcosm-diagnostics/README.md index c9e39d1f3b..7f308cf50a 100644 --- a/packages/microcosm-diagnostics/README.md +++ b/packages/microcosm-diagnostics/README.md @@ -26,3 +26,13 @@ reach validation as non-finite numbers fail construction. The writer also uses strict JSON serialization with non-finite output disabled, so a value introduced after model construction returns a structured serialization failure instead of silently becoming `null`. + +`calibration_diagnostics_v8_json_schema()` generates a Draft 2020-12 JSON +Schema for schema 8 from `CalibrationDiagnosticsV8`, for consumers that check +the file without importing these models. The models remain the authority. +Their cross-field reconciliations, finite-number and non-blank-text checks, +and strict integers cannot be written in a schema generated this way. The +schema's description lists each of these rules +(`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS`), and a test fails if a model +validator is added without being listed. Complete validation still goes +through `parse_calibration_diagnostics`. diff --git a/packages/microcosm-diagnostics/pyproject.toml b/packages/microcosm-diagnostics/pyproject.toml index 1d295cb0f9..ff95ecc001 100644 --- a/packages/microcosm-diagnostics/pyproject.toml +++ b/packages/microcosm-diagnostics/pyproject.toml @@ -13,7 +13,9 @@ Homepage = "https://microcosm.institute" Repository = "https://github.com/PolicyEngine/microcosm" [dependency-groups] -dev = ["pytest>=8"] +# jsonschema checks documents against the generated schema-8 JSON Schema in +# tests only; the package itself needs nothing beyond pydantic. +dev = ["pytest>=8", "jsonschema>=4.23,<5"] [build-system] requires = ["hatchling"] diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py b/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py index fc56769f93..24f58a56f8 100644 --- a/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py @@ -1,5 +1,14 @@ """Typed calibration diagnostics shared across the Microcosm stack.""" +from microcosm.diagnostics.json_schema import ( + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS, + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID, + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE, + JSON_SCHEMA_DRAFT_2020_12, + JsonSchemaGap, + calibration_diagnostics_v8_json_schema, + render_calibration_diagnostics_v8_json_schema, +) from microcosm.diagnostics.schema import ( CALIBRATION_DIAGNOSTICS_ADAPTER, CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION, @@ -36,6 +45,10 @@ __all__ = [ "CALIBRATION_DIAGNOSTICS_ADAPTER", "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE", + "JSON_SCHEMA_DRAFT_2020_12", "SUPPORTED_CALIBRATION_DIAGNOSTICS_SCHEMA_VERSIONS", "CalibrationDiagnostics", "CalibrationDiagnosticsV8", @@ -43,6 +56,7 @@ "DiagnosticsWriteFailure", "DiagnosticsWriteOutcome", "DiagnosticsWriteSuccess", + "JsonSchemaGap", "UK_DIAGNOSTICS_SCHEMA_VERSION", "UK_TARGET_GEOGRAPHY_LEVELS", "UKAreaFit", @@ -59,8 +73,10 @@ "UKWeightSummary", "UKZeroWeightStratum", "calibration_diagnostics_json_schema", + "calibration_diagnostics_v8_json_schema", "failed_diagnostics_outcome", "parse_calibration_diagnostics", + "render_calibration_diagnostics_v8_json_schema", "write_calibration_diagnostics", ] diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py b/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py new file mode 100644 index 0000000000..761612ef77 --- /dev/null +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py @@ -0,0 +1,281 @@ +"""A Draft 2020-12 JSON Schema for calibration diagnostics schema 8. + +The schema is generated from :class:`CalibrationDiagnosticsV8`, so it cannot +drift from the models in field names, types, bounds, enumerations, required +fields or undeclared-field refusal. The models remain the authority: some of +their rules cannot be written in a schema generated this way, and the schema's +description lists each one (:data:`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS`). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Final + +from pydantic import JsonValue + +from microcosm.diagnostics.schema import CalibrationDiagnosticsV8 + +JSON_SCHEMA_DRAFT_2020_12: Final = "https://json-schema.org/draft/2020-12/schema" +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID: Final = ( + "calibration_diagnostics.v8.schema.json" +) + + +@dataclass(frozen=True) +class JsonSchemaGap: + """A rule the schema-8 models enforce that the generated schema leaves out. + + ``validators`` names the model validator functions that enforce the rule, + by qualified name; it is empty when the rule comes from model + configuration or from how validators evaluate patterns. + """ + + key: str + rule: str + validators: tuple[str, ...] = () + + +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS: Final[tuple[JsonSchemaGap, ...]] = ( + JsonSchemaGap( + key="finite-numbers", + rule=( + "Every number is finite, at any depth, including inside options, " + "targets[].metadata and build. JSON has no NaN or Infinity, but " + "lenient parsers such as Python's json module accept those tokens, " + "and a JSON Schema number cannot exclude the values they produce, " + "so parse submissions with a parser that refuses them." + ), + validators=( + "DiagnosticsModel.reject_non_finite_numbers", + "CalibrationDiagnosticsV8.finite_optional_scalar", + ), + ), + JsonSchemaGap( + key="strict-integers", + rule=( + "Integer fields hold integers written without a fraction or " + "exponent. JSON Schema counts 2.0 and 1e3 as integers; the strict " + "models refuse them, because Python's json module decodes them as " + "floats." + ), + ), + JsonSchemaGap( + key="non-blank-text", + rule=( + "Strings the models type as non-blank contain at least one " + "character that is not whitespace. These are most names, " + "identifiers, labels, sources, entities, reasons, codes and " + "messages, and the keys of uk_diagnostics.target_observation_basis, " + "the zero-weight strata and the consumption-drift maps." + ), + validators=("_require_non_blank",), + ), + JsonSchemaGap( + key="digest-pattern-anchoring", + rule=( + "Digests match ^[0-9a-f]{64}$ with nothing after them. In the " + "ECMA-262 regular expressions JSON Schema specifies, $ matches " + "only at the end of the string, but validators that evaluate " + "pattern with Python's re module also let $ match before a final " + "newline, so they accept a digest followed by a newline." + ), + ), + JsonSchemaGap( + key="hierarchy-identity", + rule=( + "In each targets[].hierarchy, category.provider_id equals " + "provider.id and the dimensions have unique ids." + ), + validators=("CalibrationHierarchy.category_matches_provider",), + ), + JsonSchemaGap( + key="target-hierarchy", + rule="Each targets[].hierarchy.target.id equals that row's target_name.", + validators=("TargetDiagnosticV8.hierarchy_target_matches_row",), + ), + JsonSchemaGap( + key="target-surface-shape", + rule=( + "target_surface.constraint_matrix has n_targets rows and " + "n_records columns, and its nnz is at most rows times columns." + ), + validators=("TargetSurface.reconcile_matrix_shape",), + ), + JsonSchemaGap( + key="diagnostics-counts", + rule=( + "target_surface.n_targets equals the number of targets, " + "target_registry.n_specs is at least the number of targets, " + "target_surface.n_records equals n_records, and n_nonzero is at " + "most n_records." + ), + validators=("CalibrationDiagnosticsV8.reconcile_counts",), + ), + JsonSchemaGap( + key="uk-extension-presence", + rule=( + 'A UK registry (target_registry.country "uk") requires ' + "uk_diagnostics; any other registry requires it to be null or " + "absent. In a UK document, uk_diagnostics.weights.n_records, " + "effective_sample_size and top_1pct_weight_share equal the " + "top-level n_records, effective_sample_size and " + "top_1pct_weight_share. The n_targets, n_scored and n_skipped " + "values in uk_diagnostics.target_pass_rates_by_geography_level sum " + "to target_registry.n_specs, the number of targets and the number " + "of skipped targets." + ), + validators=("CalibrationDiagnosticsV8.reconcile_counts",), + ), + JsonSchemaGap( + key="uk-weight-summary", + rule=( + "In uk_diagnostics.weights, positive_weight_records plus " + "zero_weight_records equals n_records, effective_sample_size is at " + "most n_records, and ess_fraction equals effective_sample_size " + "divided by n_records. median_positive_weight and " + "max_to_median_positive_weight are null exactly when " + "positive_weight_records is 0; otherwise " + "max_to_median_positive_weight equals max_weight divided by " + "median_positive_weight." + ), + validators=("UKWeightSummary.reconcile_weight_summary",), + ), + JsonSchemaGap( + key="uk-zero-weight-strata", + rule=( + "In each uk_diagnostics.zero_weight_rows_by_stratum row, " + "positive_weight_rows plus zero_weight_rows equals rows. Across the " + "strata, rows, positive_weight_rows and zero_weight_rows sum to " + "weights.n_records, weights.positive_weight_records and " + "weights.zero_weight_records." + ), + validators=( + "UKZeroWeightStratum.reconcile_row_counts", + "UKDiagnosticsV1.reconcile_uk_diagnostics", + ), + ), + JsonSchemaGap( + key="uk-geography-pass-rates", + rule=( + "uk_diagnostics.target_pass_rates_by_geography_level has exactly " + "one row for each of national, region, country, local_authority " + "and constituency. In each row, n_scored plus n_skipped equals " + "n_targets, n_within_10pct is at most n_scored, and pass_rate " + "equals n_within_10pct divided by n_targets, or is null when " + "n_targets is 0." + ), + validators=( + "UKGeographyPassRate.reconcile_pass_rate", + "UKDiagnosticsV1.reconcile_uk_diagnostics", + ), + ), + JsonSchemaGap( + key="uk-local-fit", + rule=( + "uk_diagnostics.weakest_families and weakest_areas_by_fit are both " + "present or both absent. In weakest_families, " + "weakest_areas_by_fit.bottom_by_fit and " + "weakest_areas_by_fit.countries, n_within_10pct is at most " + "n_targets and pass_rate equals n_within_10pct divided by " + "n_targets. bottom_by_fit has at most limit rows and at most " + "n_areas_scored rows." + ), + validators=( + "UKWeakestFamily.reconcile_pass_rate", + "UKAreaFit.reconcile_pass_rate", + "UKCountryFit.reconcile_pass_rate", + "UKWeakestAreasByFit.reconcile_area_counts", + "UKDiagnosticsV1.reconcile_uk_diagnostics", + ), + ), + JsonSchemaGap( + key="uk-rotated-holdout", + rule=( + "A measured uk_diagnostics.rotated_holdout has unique grains and " + "n_folds entries in both fold_losses and folds. Its fold ids cover " + "0 to n_folds - 1, each fold's holdout_loss equals its fold_losses " + "entry, mean_holdout_loss is the mean of fold_losses and " + "worst_holdout_loss is their maximum. Each fold has " + "n_holdout_targets unique holdout_target_indices." + ), + validators=( + "UKRotatedHoldoutFold.reconcile_holdout_indices", + "UKMeasuredRotatedHoldout.reconcile_folds", + ), + ), + JsonSchemaGap( + key="uk-consumption-drift", + rule=( + "Each uk_diagnostics.consumption_drift value has change equal to " + "final minus design, and its relative_change is null exactly when " + "design is 0." + ), + validators=("UKDriftValue.reconcile_change",), + ), +) + +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE: Final = ( + "Literal fields hold exactly their declared JSON value. The models compare " + "literals with Python equality, so their parser also accepts 1 for true " + "and true for 1, for example in rotated_holdout.skipped or " + "uk_diagnostics.schema_version. The canonical writer always emits the " + "declared value, which this schema requires." +) + +_TITLE: Final = "Microcosm calibration diagnostics, schema 8" +_COMMENT: Final = ( + "Generated from microcosm.diagnostics.CalibrationDiagnosticsV8 by " + "microcosm.diagnostics.calibration_diagnostics_v8_json_schema(); " + "regenerate it rather than editing it." +) + + +def _description() -> str: + gap_lines = "\n".join( + f"- {gap.rule}" for gap in CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + ) + return ( + "The calibration_diagnostics.json document, schema 8, that Microcosm " + "builds write beside a calibrated dataset. The pydantic models it is " + "generated from remain the authority. A document that satisfies this " + "schema can still break one of the rules below, which the models " + "enforce and this schema does not express, so validate with " + "microcosm.diagnostics.parse_calibration_diagnostics for the complete " + "contract. Equalities between floating-point values hold within the " + "models' tolerances.\n\n" + "Rules the models enforce that this schema does not express:\n" + f"{gap_lines}\n\n" + "Where this schema is stricter than the models' parser:\n" + f"- {CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE}" + ) + + +def calibration_diagnostics_v8_json_schema() -> dict[str, JsonValue]: + """Return the schema-8 JSON Schema generated from the typed models.""" + + generated = CalibrationDiagnosticsV8.model_json_schema(mode="validation") + generated.pop("title", None) + generated.pop("description", None) + return { + "$schema": JSON_SCHEMA_DRAFT_2020_12, + "$id": CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID, + "$comment": _COMMENT, + "title": _TITLE, + "description": _description(), + **generated, + } + + +def render_calibration_diagnostics_v8_json_schema() -> str: + """Return the schema as the exact text committed to the repository.""" + + return ( + json.dumps( + calibration_diagnostics_v8_json_schema(), + indent=2, + ensure_ascii=False, + ) + + "\n" + ) diff --git a/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py new file mode 100644 index 0000000000..abb040d1a8 --- /dev/null +++ b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py @@ -0,0 +1,619 @@ +"""The JSON Schema generated for calibration diagnostics schema 8. + +The pydantic models stay the authority. These tests pin what the generated +schema accepts and refuses, and keep its list of unexpressed model rules +complete: every validator in the schema-8 model tree must be listed. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from functools import cache + +import pytest +from jsonschema import Draft202012Validator +from pydantic import ValidationError + +from microcosm import diagnostics +from test_support.microcosm_diagnostics.json_schema import ( + json_schema_errors, + load_strict_json, +) +from test_support.microcosm_diagnostics.payloads import ( + schema_8_payload, + uk_schema_8_payload, +) + +DRAFT_2020_12 = "https://json-schema.org/draft/2020-12/schema" + +DocumentBuilder = Callable[[], object] +DocumentMutation = Callable[[dict], None] + + +@cache +def _generated_schema_text() -> str: + return json.dumps(diagnostics.calibration_diagnostics_v8_json_schema()) + + +def _generated_schema() -> dict: + """A fresh copy per test, so no test can change another's schema.""" + + return json.loads(_generated_schema_text()) + + +def _set_value(path: tuple[object, ...], value: object) -> DocumentMutation: + def mutate(document: dict) -> None: + owner = document + for part in path[:-1]: + owner = owner[part] + owner[path[-1]] = value + + return mutate + + +def _delete_value(path: tuple[object, ...]) -> DocumentMutation: + def mutate(document: dict) -> None: + owner = document + for part in path[:-1]: + owner = owner[part] + del owner[path[-1]] + + return mutate + + +def _document(builder: DocumentBuilder, mutation: DocumentMutation | None) -> object: + document = builder() + if mutation is not None: + mutation(document) + return document + + +def _payload_with_every_optional_block() -> dict: + payload = schema_8_payload() + target = payload["targets"][0] + target["period"] = "2024" + target["filter"] = {"kind": "callable", "name": "is_adult"} + target["metadata"] = {"ledger": {"values": [1, 2.5, None, True, "text"]}} + target["hierarchy"]["dimensions"] = [ + {"id": "sex", "label": "Sex", "value_id": "female", "value_label": "Female"} + ] + target.update( + { + "registry": { + "family": "census_population", + "se": None, + "signed": False, + "notes": "", + }, + "target_loss_weight": 1.0, + "target_loss_weight_share": 1.0, + "target_loss_scale": 2.0, + "final_capped_scaled_error": 0.0, + "final_loss_contribution": 0.0, + } + ) + payload["target_registry"]["n_specs"] = 2 + payload["l0_lambda"] = None + payload["loss_trajectory"] = [0.1, None, 0.01] + payload["skipped"] = [{"name": "ghost@2024", "reason": "measure is missing"}] + payload["diagnostic_warnings"] = [ + { + "code": "target_loss_attribution_withheld", + "severity": "warning", + "message": "Attribution could not be validated.", + } + ] + payload["past_cap_census"] = { + "cap": 1.0, + "scale_basis": "target", + "n_targets": 1, + "initial_past_cap": 1, + "final_past_cap": 0, + "escaped": 1, + "frozen": 0, + "pushed_out": 0, + "pushed_out_rows": [ + {"name": "population@2024", "init_rel": 1.5, "final_rel": 0.0} + ], + } + payload["target_loss_basis"] = { + "formula": "sum(weight * min(scaled_error, cap) ** 2)", + "cap": 1.0, + "target_count": 1, + "total_target_weight": 1.0, + "weight_kind": "provided", + "scale_kind": "target", + "hash_algorithm": "sha256", + "sha256": "b" * 64, + } + payload["build"] = {"release_id": "fixture", "timing": {"seconds": 1.5}} + return payload + + +def _uk_payload_with_skipped_holdout() -> dict: + payload = uk_schema_8_payload() + payload["uk_diagnostics"]["rotated_holdout"] = {"skipped": True} + return payload + + +def _legacy_schema_6_payload() -> dict: + """A document the parser accepts through its schema-6 compatibility branch.""" + + payload = schema_8_payload() + payload["schema_version"] = 6 + del payload["targets"][0]["hierarchy"] + return payload + + +def _array_document() -> list: + return [] + + +def test_generated_schema_declares_draft_2020_12_and_passes_its_metaschema() -> None: + schema = _generated_schema() + + assert schema["$schema"] == DRAFT_2020_12 + assert schema["$id"] == "calibration_diagnostics.v8.schema.json" + assert schema["title"] == "Microcosm calibration diagnostics, schema 8" + assert diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID == schema["$id"] + Draft202012Validator.check_schema(schema) + + +def test_generated_schema_pins_version_8_and_forbids_undeclared_fields() -> None: + schema = _generated_schema() + object_definitions = { + name: definition + for name, definition in schema["$defs"].items() + if definition.get("type") == "object" + } + + assert schema["properties"]["schema_version"]["const"] == 8 + assert schema["additionalProperties"] is False + assert "UKDiagnosticsV1" in object_definitions + assert all( + definition["additionalProperties"] is False + for definition in object_definitions.values() + ) + + +@pytest.mark.parametrize( + "builder", + [ + pytest.param(schema_8_payload, id="minimal-us-registry"), + pytest.param(_payload_with_every_optional_block, id="every-optional-block"), + pytest.param(uk_schema_8_payload, id="complete-uk-extension"), + pytest.param(_uk_payload_with_skipped_holdout, id="skipped-uk-holdout"), + ], +) +def test_generated_schema_accepts_schema_8_documents( + builder: DocumentBuilder, +) -> None: + document = load_strict_json(json.dumps(builder(), allow_nan=False)) + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + assert json_schema_errors(document, _generated_schema()) == [] + + +@pytest.mark.parametrize( + ("builder", "mutation", "error_path"), + [ + pytest.param( + schema_8_payload, + _delete_value(("targets",)), + "$", + id="missing-required-field", + ), + pytest.param( + schema_8_payload, + _set_value(("unexpected",), 1), + "$", + id="undeclared-top-level-field", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "weights", "unexpected"), 1), + "$.uk_diagnostics.weights", + id="undeclared-uk-field", + ), + pytest.param( + schema_8_payload, + _set_value(("n_records",), "2"), + "$.n_records", + id="string-for-integer", + ), + pytest.param( + schema_8_payload, + _set_value(("n_nonzero",), True), + "$.n_nonzero", + id="boolean-for-integer", + ), + pytest.param( + schema_8_payload, + _set_value(("final_loss",), "0.01"), + "$.final_loss", + id="string-for-number", + ), + pytest.param( + schema_8_payload, + _set_value(("n_nonzero",), -1), + "$.n_nonzero", + id="negative-count", + ), + pytest.param( + schema_8_payload, + _set_value(("target_registry", "n_specs"), 0), + "$.target_registry.n_specs", + id="zero-for-positive-count", + ), + pytest.param( + schema_8_payload, + _set_value(("schema_version",), 7), + "$.schema_version", + id="schema-version-7", + ), + pytest.param( + schema_8_payload, + _set_value(("target_surface", "sha256"), "A" * 64), + "$.target_surface.sha256", + id="uppercase-digest", + ), + pytest.param( + schema_8_payload, + _set_value(("targets", 0, "measure", "kind"), "expression"), + "$.targets[0].measure.kind", + id="unknown-selector-kind", + ), + pytest.param( + uk_schema_8_payload, + _set_value( + ( + "uk_diagnostics", + "target_pass_rates_by_geography_level", + 0, + "geography_level", + ), + "ward", + ), + "$.uk_diagnostics.target_pass_rates_by_geography_level[0].geography_level", + id="unknown-uk-geography-level", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "weights", "ess_fraction"), 1.5), + "$.uk_diagnostics.weights.ess_fraction", + id="fraction-above-one", + ), + pytest.param( + schema_8_payload, + _set_value(("weight_entity",), None), + "$.weight_entity", + id="null-for-required-text", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "zero_weight_rows_by_stratum"), []), + "$.uk_diagnostics.zero_weight_rows_by_stratum", + id="empty-required-list", + ), + pytest.param( + uk_schema_8_payload, + _set_value(("uk_diagnostics", "rotated_holdout", "n_folds"), 1), + "$.uk_diagnostics.rotated_holdout.n_folds", + id="too-few-holdout-folds", + ), + pytest.param( + schema_8_payload, + _set_value(("targets", 0, "period"), 2024.5), + "$.targets[0].period", + id="fractional-period", + ), + pytest.param( + schema_8_payload, + _set_value(("loss_trajectory",), ["0.1"]), + "$.loss_trajectory[0]", + id="string-in-loss-trajectory", + ), + pytest.param( + _legacy_schema_6_payload, + None, + "$.schema_version", + id="legacy-schema-6-document", + ), + pytest.param(_array_document, None, "$", id="array-document"), + ], +) +def test_generated_schema_and_models_reject_invalid_documents( + builder: DocumentBuilder, + mutation: DocumentMutation | None, + error_path: str, +) -> None: + document = _document(builder, mutation) + + errors = json_schema_errors(document, _generated_schema()) + + assert any(error.startswith(f"{error_path}: ") for error in errors), errors + with pytest.raises(ValidationError): + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + +_DUPLICATED_DIMENSION = { + "id": "sex", + "label": "Sex", + "value_id": "female", + "value_label": "Female", +} + +#: One or more documents per documented gap: the generated schema accepts +#: each, and the models refuse it with the given message. +GAP_CASES = [ + pytest.param( + "finite-numbers", + schema_8_payload, + _set_value(("final_loss",), float("nan")), + "must be finite or null", + id="finite-numbers-top-level-scalar", + ), + pytest.param( + "finite-numbers", + schema_8_payload, + _set_value(("options", "nested"), [{"value": float("inf")}]), + "must be finite or null", + id="finite-numbers-nested-solver-option", + ), + pytest.param( + "strict-integers", + schema_8_payload, + _set_value(("n_records",), 2.0), + "valid integer", + id="strict-integers", + ), + pytest.param( + "non-blank-text", + schema_8_payload, + _set_value(("targets", 0, "source"), " "), + "non-whitespace", + id="non-blank-text", + ), + pytest.param( + "digest-pattern-anchoring", + schema_8_payload, + _set_value(("target_surface", "sha256"), "a" * 64 + "\n"), + "should match pattern", + id="digest-pattern-anchoring", + ), + pytest.param( + "hierarchy-identity", + schema_8_payload, + _set_value( + ("targets", 0, "hierarchy", "category", "provider_id"), + "another-provider", + ), + "provider_id must match provider id", + id="hierarchy-category-provider", + ), + pytest.param( + "hierarchy-identity", + schema_8_payload, + _set_value( + ("targets", 0, "hierarchy", "dimensions"), + [_DUPLICATED_DIMENSION, _DUPLICATED_DIMENSION], + ), + "dimension ids must be unique", + id="hierarchy-dimension-ids", + ), + pytest.param( + "target-hierarchy", + schema_8_payload, + _set_value(("targets", 0, "hierarchy", "target", "id"), "wrong-target"), + "hierarchy target id must match target_name", + id="target-hierarchy", + ), + pytest.param( + "target-surface-shape", + schema_8_payload, + _set_value(("target_surface", "constraint_matrix", "rows"), 2), + "rows must equal n_targets", + id="target-surface-shape", + ), + pytest.param( + "diagnostics-counts", + schema_8_payload, + _set_value(("n_nonzero",), 3), + "n_nonzero cannot exceed n_records", + id="diagnostics-counts", + ), + pytest.param( + "uk-extension-presence", + uk_schema_8_payload, + _delete_value(("uk_diagnostics",)), + "require a uk_diagnostics block", + id="uk-registry-without-uk-extension", + ), + pytest.param( + "uk-extension-presence", + uk_schema_8_payload, + _set_value(("target_registry", "country"), "us"), + "only valid when target_registry.country is 'uk'", + id="uk-extension-on-another-registry", + ), + pytest.param( + "uk-weight-summary", + uk_schema_8_payload, + _set_value(("uk_diagnostics", "weights", "ess_fraction"), 0.25), + "ess_fraction must equal", + id="uk-weight-summary", + ), + pytest.param( + "uk-zero-weight-strata", + uk_schema_8_payload, + _set_value(("uk_diagnostics", "zero_weight_rows_by_stratum", 0, "rows"), 3), + "stratum row counts do not reconcile", + id="uk-zero-weight-strata", + ), + pytest.param( + "uk-geography-pass-rates", + uk_schema_8_payload, + _set_value( + ( + "uk_diagnostics", + "target_pass_rates_by_geography_level", + 4, + "geography_level", + ), + "national", + ), + "levels must be unique", + id="uk-geography-pass-rates", + ), + pytest.param( + "uk-local-fit", + uk_schema_8_payload, + _delete_value(("uk_diagnostics", "weakest_areas_by_fit")), + "require both family and area summaries", + id="uk-local-fit", + ), + pytest.param( + "uk-rotated-holdout", + uk_schema_8_payload, + _set_value(("uk_diagnostics", "rotated_holdout", "fold_losses"), [0.1, 0.3]), + "fold rows must match fold_losses", + id="uk-rotated-holdout", + ), + pytest.param( + "uk-consumption-drift", + uk_schema_8_payload, + _set_value( + ( + "uk_diagnostics", + "consumption_drift", + "totals", + "petrol_spending", + "change", + ), + 3.0, + ), + "final minus design", + id="uk-consumption-drift", + ), +] + + +@pytest.mark.parametrize( + ("gap_key", "builder", "mutation", "model_error"), + GAP_CASES, +) +def test_generated_schema_accepts_documents_only_the_models_reject( + gap_key: str, + builder: DocumentBuilder, + mutation: DocumentMutation, + model_error: str, +) -> None: + documented_keys = { + gap.key for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + } + document = _document(builder, mutation) + + assert gap_key in documented_keys + assert json_schema_errors(document, _generated_schema()) == [] + with pytest.raises(ValidationError, match=model_error): + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + +def test_every_documented_gap_has_a_characterizing_document() -> None: + documented_keys = [ + gap.key for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + ] + characterized_keys = {case.values[0] for case in GAP_CASES} + + assert len(documented_keys) == len(set(documented_keys)) + assert set(documented_keys) == characterized_keys + + +@pytest.mark.parametrize( + ("mutation", "error_path"), + [ + pytest.param( + _set_value(("uk_diagnostics", "rotated_holdout"), {"skipped": 1}), + "$.uk_diagnostics.rotated_holdout", + id="one-for-true", + ), + pytest.param( + _set_value(("uk_diagnostics", "schema_version"), True), + "$.uk_diagnostics.schema_version", + id="true-for-one", + ), + ], +) +def test_generated_schema_requires_the_literal_values_the_writer_emits( + mutation: DocumentMutation, + error_path: str, +) -> None: + document = _document(uk_schema_8_payload, mutation) + + errors = json_schema_errors(document, _generated_schema()) + model = diagnostics.CalibrationDiagnosticsV8.model_validate(document) + rewritten = json.loads(model.model_dump_json(exclude_defaults=True)) + + assert any(error.startswith(f"{error_path}: ") for error in errors), errors + assert json_schema_errors(rewritten, _generated_schema()) == [] + + +def _validator_qualified_names(core_schema: object) -> set[str]: + """Every validator function pydantic runs for a model, by qualified name.""" + + names: set[str] = set() + pending = [core_schema] + visited: set[int] = set() + while pending: + node = pending.pop() + if id(node) in visited: + continue + visited.add(id(node)) + if isinstance(node, dict): + if str(node.get("type", "")).startswith("function-"): + names.add(node["function"]["function"].__qualname__) + pending.extend(node.values()) + elif isinstance(node, (list, tuple)): + pending.extend(node) + return names + + +def test_every_model_validator_is_documented_as_a_schema_gap() -> None: + model_validators = _validator_qualified_names( + diagnostics.CalibrationDiagnosticsV8.__pydantic_core_schema__ + ) + documented_validators = { + validator + for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS + for validator in gap.validators + } + + assert "CalibrationDiagnosticsV8.reconcile_counts" in model_validators + assert documented_validators == model_validators + + +def test_generated_schema_description_lists_every_gap() -> None: + description = _generated_schema()["description"] + + assert "microcosm.diagnostics.parse_calibration_diagnostics" in description + assert diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE in ( + description + ) + for gap in diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS: + assert gap.rule in description, gap.key + + +def test_rendered_schema_is_deterministic_json_text() -> None: + rendered = diagnostics.render_calibration_diagnostics_v8_json_schema() + + assert rendered == diagnostics.render_calibration_diagnostics_v8_json_schema() + assert rendered.endswith("}\n") + assert json.loads(rendered) == _generated_schema() + assert list(json.loads(rendered))[:5] == [ + "$schema", + "$id", + "$comment", + "title", + "description", + ] diff --git a/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py b/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py index 399b6ab108..864afdf3f7 100644 --- a/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py +++ b/packages/microcosm-diagnostics/tests/engine_free/shared/test_schema.py @@ -12,218 +12,10 @@ parse_calibration_diagnostics, write_calibration_diagnostics, ) - - -def _payload() -> dict: - digest = "a" * 64 - return { - "schema_version": 8, - "weight_entity": "household", - "options": {"epochs": 1}, - "target_surface": { - "schema_version": 1, - "weight_entity": "household", - "n_targets": 1, - "n_records": 2, - "constraint_matrix": {"rows": 1, "columns": 2, "nnz": 2}, - "sha256": digest, - "names_sha256": digest, - "values_sha256": digest, - }, - "target_registry": {"country": "us", "version": digest, "n_specs": 1}, - "l0_lambda": 0.0, - "n_nonzero": 2, - "n_records": 2, - "initial_loss": 0.1, - "final_loss": 0.01, - "fraction_within_10pct": 1.0, - "effective_sample_size": 2.0, - "realized_max_weight_ratio": 1.0, - "top_1pct_weight_share": 0.5, - "loss_trajectory": [0.1], - "skipped": [], - "past_cap_census": None, - "diagnostic_warnings": [], - "targets": [ - { - "name": "population@2024", - "target_name": "population", - "period": 2024, - "entity": "household", - "measure": {"kind": "column", "name": "household_count"}, - "filter": None, - "source": "Census fixture", - "metadata": {}, - "target": 2.0, - "compiled_target": 2.0, - "initial_estimate": 1.8, - "final_estimate": 2.0, - "relative_error": 0.0, - "within_tolerance": None, - "hierarchy": { - "provider": {"id": "census", "label": "Census"}, - "category": { - "id": "census.population", - "label": "Population", - "provider_id": "census", - }, - "geography": { - "id": "0100000US", - "label": "United States", - "level": "country", - }, - "dimensions": [], - "target": {"id": "population", "label": "Population"}, - }, - } - ], - } - - -def _uk_payload() -> dict: - payload = _payload() - payload["target_registry"]["country"] = "uk" - payload["n_nonzero"] = 1 - payload["effective_sample_size"] = 1.0 - payload["top_1pct_weight_share"] = 1.0 - payload["uk_diagnostics"] = { - "schema_version": 1, - "weights": { - "n_records": 2, - "positive_weight_records": 1, - "zero_weight_records": 1, - "total_weight": 1.0, - "effective_sample_size": 1.0, - "ess_fraction": 0.5, - "median_positive_weight": 1.0, - "max_weight": 1.0, - "max_to_median_positive_weight": 1.0, - "top_1pct_weight_share": 1.0, - }, - "zero_weight_rows_by_stratum": [ - { - "stratum": {"source": "frs"}, - "rows": 2, - "positive_weight_rows": 1, - "zero_weight_rows": 1, - "weight_sum": 1.0, - } - ], - "target_pass_rates_by_geography_level": [ - { - "geography_level": level, - "n_targets": 1 if level == "national" else 0, - "n_scored": 1 if level == "national" else 0, - "n_skipped": 0, - "n_within_10pct": 1 if level == "national" else 0, - "pass_rate": 1.0 if level == "national" else None, - } - for level in ( - "national", - "region", - "country", - "local_authority", - "constituency", - ) - ], - "target_observation_basis": {"population@2024": "annual_stock"}, - "weakest_families": [ - { - "family": "population", - "n_targets": 1, - "n_within_10pct": 1, - "pass_rate": 1.0, - "worst_target": "population@2024", - "worst_abs_relative_error": 0.0, - "loss_contribution": 0.0, - "loss_share": 0.0, - } - ], - "weakest_areas_by_fit": { - "limit": 15, - "n_areas_scored": 1, - "bottom_by_fit": [ - { - "geography_level": "constituency", - "area_code": "E14000001", - "country": "England", - "n_targets": 1, - "n_within_10pct": 1, - "pass_rate": 1.0, - "worst_target": "population@2024", - "worst_abs_relative_error": 0.0, - "loss_contribution": 0.0, - "support_below_floor": True, - } - ], - "countries": [ - { - "country": "England", - "geography_level": "constituency", - "n_areas": 1, - "n_targets": 1, - "n_within_10pct": 1, - "pass_rate": 1.0, - "worst_target": "population@2024", - "worst_abs_relative_error": 0.0, - "loss_contribution": 0.0, - } - ], - }, - "rotated_holdout": { - "report_only": True, - "method": "rotated_folds", - "target_loss_cap": 10.0, - "loss_weight_scale": "held_local_grains_only", - "target_weight_rule": "grain_equal", - "population": "held_out_local_targets", - "grains": ["constituency", "local_authority"], - "n_folds": 2, - "seed": 20260529, - "solve_seed": 7, - "mean_holdout_loss": 0.15, - "worst_holdout_loss": 0.2, - "fold_losses": [0.1, 0.2], - "folds": [ - { - "fold": 0, - "n_train_targets": 1, - "n_holdout_targets": 1, - "holdout_target_indices": [0], - "training_national_rows": 1, - "holdout_loss": 0.1, - }, - { - "fold": 1, - "n_train_targets": 1, - "n_holdout_targets": 1, - "holdout_target_indices": [1], - "training_national_rows": 1, - "holdout_loss": 0.2, - }, - ], - }, - "consumption_drift": { - "report_only": True, - "totals": { - "petrol_spending": { - "design": 100.0, - "final": 102.0, - "change": 2.0, - "relative_change": 0.02, - } - }, - "shares": { - "fuel_car_households_without_fuel": { - "design": 0.0, - "final": 0.0, - "change": 0.0, - "relative_change": None, - } - }, - }, - } - return payload +from test_support.microcosm_diagnostics.payloads import ( + schema_8_payload, + uk_schema_8_payload, +) def _delete_path(payload: dict, path: tuple[object, ...]) -> None: @@ -234,14 +26,16 @@ def _delete_path(payload: dict, path: tuple[object, ...]) -> None: def test_schema_8_round_trips_as_a_typed_model() -> None: - model = parse_calibration_diagnostics(_payload()) + model = parse_calibration_diagnostics(schema_8_payload()) assert isinstance(model, CalibrationDiagnosticsV8) - assert json.loads(model.model_dump_json(exclude_defaults=True)) == _payload() + assert ( + json.loads(model.model_dump_json(exclude_defaults=True)) == schema_8_payload() + ) assert calibration_diagnostics_json_schema()["$defs"] def test_schema_8_round_trips_the_complete_uk_extension() -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() model = parse_calibration_diagnostics(payload) assert isinstance(model, CalibrationDiagnosticsV8) @@ -252,7 +46,7 @@ def test_uk_area_fit_still_parses_legacy_support_figures() -> None: # Released diagnostics carry the floor flag in place of an area's support # figures (microcosm#1301); payloads written before still parse, and a # row may carry neither. - payload = _uk_payload() + payload = uk_schema_8_payload() (area,) = payload["uk_diagnostics"]["weakest_areas_by_fit"]["bottom_by_fit"] del area["support_below_floor"] bare = parse_calibration_diagnostics(payload) @@ -336,7 +130,7 @@ def test_every_declared_uk_record_forbids_undeclared_fields() -> None: def test_schema_8_rejects_missing_fields_across_the_uk_extension( field_path: tuple[object, ...], ) -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() _delete_path(payload, field_path) with pytest.raises(ValidationError): @@ -368,7 +162,7 @@ def test_schema_8_rejects_missing_fields_across_the_uk_extension( def test_schema_8_rejects_undeclared_fields_across_the_uk_extension( field_path: tuple[object, ...], ) -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() owner = payload for part in field_path: owner = owner[part] @@ -379,7 +173,7 @@ def test_schema_8_rejects_undeclared_fields_across_the_uk_extension( def test_schema_8_requires_the_typed_uk_extension_for_uk_registries() -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() del payload["uk_diagnostics"] with pytest.raises(ValidationError, match="require a uk_diagnostics block"): @@ -387,7 +181,7 @@ def test_schema_8_requires_the_typed_uk_extension_for_uk_registries() -> None: def test_schema_8_schematizes_an_explicitly_skipped_uk_holdout() -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() payload["uk_diagnostics"]["rotated_holdout"] = {"skipped": True} model = parse_calibration_diagnostics(payload) @@ -399,7 +193,7 @@ def test_schema_8_schematizes_an_explicitly_skipped_uk_holdout() -> None: def test_schema_8_rejects_incomplete_hierarchy() -> None: - payload = _payload() + payload = schema_8_payload() del payload["targets"][0]["hierarchy"] with pytest.raises(ValidationError, match="hierarchy"): parse_calibration_diagnostics(payload) @@ -425,7 +219,7 @@ def test_schema_8_rejects_inconsistent_hierarchy_identity( value: str, message: str, ) -> None: - payload = _payload() + payload = schema_8_payload() owner = payload for part in field_path[:-1]: owner = owner[part] @@ -444,7 +238,7 @@ def test_schema_8_rejects_inconsistent_hierarchy_identity( ], ) def test_schema_8_rejects_whitespace_only_text(field_path: tuple[object, ...]) -> None: - payload = _payload() + payload = schema_8_payload() payload["targets"][0]["hierarchy"]["dimensions"] = [ {"id": "sex", "label": "Sex", "value_id": "female", "value_label": "Female"} ] @@ -465,7 +259,7 @@ def test_schema_8_requires_positive_declared_target_counts( section: str, field: str, ) -> None: - payload = _payload() + payload = schema_8_payload() payload[section][field] = 0 with pytest.raises(ValidationError, match="greater than 0"): @@ -473,7 +267,7 @@ def test_schema_8_requires_positive_declared_target_counts( def test_schema_8_reconciles_target_surface_matrix_shape() -> None: - payload = _payload() + payload = schema_8_payload() payload["target_surface"]["constraint_matrix"]["rows"] = 2 with pytest.raises(ValidationError, match="rows must equal n_targets"): @@ -501,7 +295,9 @@ def test_schema_8_rejects_non_finite_numbers_recursively( location: str, value: float, ) -> None: - payload = _uk_payload() if location == "UK extension" else _payload() + payload = ( + uk_schema_8_payload() if location == "UK extension" else schema_8_payload() + ) if location == "top-level scalar": payload["final_loss"] = value elif location == "target scalar": @@ -524,7 +320,7 @@ def test_schema_8_rejects_non_finite_numbers_recursively( def test_writer_strict_serialization_rejects_a_non_finite_mutated_mapping( tmp_path: Path, ) -> None: - payload = _payload() + payload = schema_8_payload() payload["build"] = {"nested": {"value": 1.0}} model = CalibrationDiagnosticsV8.model_validate(payload) assert model.build is not None @@ -546,7 +342,7 @@ def test_writer_is_atomic_and_removes_stale_output_on_failure( ) -> None: path = tmp_path / "calibration_diagnostics.json" path.write_text("stale") - model = CalibrationDiagnosticsV8.model_validate(_payload()) + model = CalibrationDiagnosticsV8.model_validate(schema_8_payload()) def fail_write(self: Path, data: bytes) -> int: raise OSError("synthetic write failure") @@ -565,7 +361,7 @@ def test_unexpected_writer_failure_does_not_escape( ) -> None: path = tmp_path / "calibration_diagnostics.json" path.write_text("stale") - model = CalibrationDiagnosticsV8.model_validate(_payload()) + model = CalibrationDiagnosticsV8.model_validate(schema_8_payload()) def fail_write(self: Path, data: bytes) -> int: raise RuntimeError("synthetic unexpected failure") @@ -579,7 +375,7 @@ def fail_write(self: Path, data: bytes) -> int: def test_writer_round_trips_valid_output(tmp_path: Path) -> None: - model = CalibrationDiagnosticsV8.model_validate(_payload()) + model = CalibrationDiagnosticsV8.model_validate(schema_8_payload()) path = tmp_path / "calibration_diagnostics.json" outcome = write_calibration_diagnostics(model, path) @@ -595,7 +391,7 @@ def test_writer_round_trips_valid_output(tmp_path: Path) -> None: ], ) def test_uk_consumption_drift_values_reconcile(change: dict, match: str) -> None: - payload = _uk_payload() + payload = uk_schema_8_payload() payload["uk_diagnostics"]["consumption_drift"]["totals"]["petrol_spending"].update( change ) diff --git a/test_support/microcosm_diagnostics/__init__.py b/test_support/microcosm_diagnostics/__init__.py new file mode 100644 index 0000000000..85e1acbbbc --- /dev/null +++ b/test_support/microcosm_diagnostics/__init__.py @@ -0,0 +1 @@ +"""Shared support for microcosm-diagnostics tests.""" diff --git a/test_support/microcosm_diagnostics/json_schema.py b/test_support/microcosm_diagnostics/json_schema.py new file mode 100644 index 0000000000..062827e848 --- /dev/null +++ b/test_support/microcosm_diagnostics/json_schema.py @@ -0,0 +1,38 @@ +"""Check calibration-diagnostics documents the way a JSON Schema intake would.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Iterator, Mapping + +from jsonschema import Draft202012Validator, ValidationError + + +def _refuse_non_finite_literal(literal: str) -> float: + raise ValueError(f"strict JSON has no {literal} literal") + + +def load_strict_json(text: str | bytes) -> object: + """Decode RFC 8259 JSON, refusing the NaN and Infinity tokens Python accepts.""" + + return json.loads(text, parse_constant=_refuse_non_finite_literal) + + +def _leaf_errors(errors: Iterable[ValidationError]) -> Iterator[ValidationError]: + """Descend into ``anyOf`` branches so each error names its innermost path.""" + + for error in errors: + if error.context: + yield from _leaf_errors(error.context) + else: + yield error + + +def json_schema_errors(document: object, schema: Mapping[str, object]) -> list[str]: + """Return every ``: `` error, sorted for stable output.""" + + validator = Draft202012Validator(schema) + return sorted( + f"{error.json_path}: {error.message}" + for error in _leaf_errors(validator.iter_errors(document)) + ) diff --git a/test_support/microcosm_diagnostics/payloads.py b/test_support/microcosm_diagnostics/payloads.py new file mode 100644 index 0000000000..30b85fbbee --- /dev/null +++ b/test_support/microcosm_diagnostics/payloads.py @@ -0,0 +1,223 @@ +"""Valid calibration-diagnostics schema-8 documents shared by tests. + +Each call returns a fresh, fully reconciled document, so a test can mutate one +field and know that any refusal comes from that change. +""" + +from __future__ import annotations + + +def schema_8_payload() -> dict: + """A minimal US-registry document with one target and no optional blocks.""" + + digest = "a" * 64 + return { + "schema_version": 8, + "weight_entity": "household", + "options": {"epochs": 1}, + "target_surface": { + "schema_version": 1, + "weight_entity": "household", + "n_targets": 1, + "n_records": 2, + "constraint_matrix": {"rows": 1, "columns": 2, "nnz": 2}, + "sha256": digest, + "names_sha256": digest, + "values_sha256": digest, + }, + "target_registry": {"country": "us", "version": digest, "n_specs": 1}, + "l0_lambda": 0.0, + "n_nonzero": 2, + "n_records": 2, + "initial_loss": 0.1, + "final_loss": 0.01, + "fraction_within_10pct": 1.0, + "effective_sample_size": 2.0, + "realized_max_weight_ratio": 1.0, + "top_1pct_weight_share": 0.5, + "loss_trajectory": [0.1], + "skipped": [], + "past_cap_census": None, + "diagnostic_warnings": [], + "targets": [ + { + "name": "population@2024", + "target_name": "population", + "period": 2024, + "entity": "household", + "measure": {"kind": "column", "name": "household_count"}, + "filter": None, + "source": "Census fixture", + "metadata": {}, + "target": 2.0, + "compiled_target": 2.0, + "initial_estimate": 1.8, + "final_estimate": 2.0, + "relative_error": 0.0, + "within_tolerance": None, + "hierarchy": { + "provider": {"id": "census", "label": "Census"}, + "category": { + "id": "census.population", + "label": "Population", + "provider_id": "census", + }, + "geography": { + "id": "0100000US", + "label": "United States", + "level": "country", + }, + "dimensions": [], + "target": {"id": "population", "label": "Population"}, + }, + } + ], + } + + +def uk_schema_8_payload() -> dict: + """The minimal document on a UK registry, carrying every UK extension block.""" + + payload = schema_8_payload() + payload["target_registry"]["country"] = "uk" + payload["n_nonzero"] = 1 + payload["effective_sample_size"] = 1.0 + payload["top_1pct_weight_share"] = 1.0 + payload["uk_diagnostics"] = { + "schema_version": 1, + "weights": { + "n_records": 2, + "positive_weight_records": 1, + "zero_weight_records": 1, + "total_weight": 1.0, + "effective_sample_size": 1.0, + "ess_fraction": 0.5, + "median_positive_weight": 1.0, + "max_weight": 1.0, + "max_to_median_positive_weight": 1.0, + "top_1pct_weight_share": 1.0, + }, + "zero_weight_rows_by_stratum": [ + { + "stratum": {"source": "frs"}, + "rows": 2, + "positive_weight_rows": 1, + "zero_weight_rows": 1, + "weight_sum": 1.0, + } + ], + "target_pass_rates_by_geography_level": [ + { + "geography_level": level, + "n_targets": 1 if level == "national" else 0, + "n_scored": 1 if level == "national" else 0, + "n_skipped": 0, + "n_within_10pct": 1 if level == "national" else 0, + "pass_rate": 1.0 if level == "national" else None, + } + for level in ( + "national", + "region", + "country", + "local_authority", + "constituency", + ) + ], + "target_observation_basis": {"population@2024": "annual_stock"}, + "weakest_families": [ + { + "family": "population", + "n_targets": 1, + "n_within_10pct": 1, + "pass_rate": 1.0, + "worst_target": "population@2024", + "worst_abs_relative_error": 0.0, + "loss_contribution": 0.0, + "loss_share": 0.0, + } + ], + "weakest_areas_by_fit": { + "limit": 15, + "n_areas_scored": 1, + "bottom_by_fit": [ + { + "geography_level": "constituency", + "area_code": "E14000001", + "country": "England", + "n_targets": 1, + "n_within_10pct": 1, + "pass_rate": 1.0, + "worst_target": "population@2024", + "worst_abs_relative_error": 0.0, + "loss_contribution": 0.0, + "support_below_floor": True, + } + ], + "countries": [ + { + "country": "England", + "geography_level": "constituency", + "n_areas": 1, + "n_targets": 1, + "n_within_10pct": 1, + "pass_rate": 1.0, + "worst_target": "population@2024", + "worst_abs_relative_error": 0.0, + "loss_contribution": 0.0, + } + ], + }, + "rotated_holdout": { + "report_only": True, + "method": "rotated_folds", + "target_loss_cap": 10.0, + "loss_weight_scale": "held_local_grains_only", + "target_weight_rule": "grain_equal", + "population": "held_out_local_targets", + "grains": ["constituency", "local_authority"], + "n_folds": 2, + "seed": 20260529, + "solve_seed": 7, + "mean_holdout_loss": 0.15, + "worst_holdout_loss": 0.2, + "fold_losses": [0.1, 0.2], + "folds": [ + { + "fold": 0, + "n_train_targets": 1, + "n_holdout_targets": 1, + "holdout_target_indices": [0], + "training_national_rows": 1, + "holdout_loss": 0.1, + }, + { + "fold": 1, + "n_train_targets": 1, + "n_holdout_targets": 1, + "holdout_target_indices": [1], + "training_national_rows": 1, + "holdout_loss": 0.2, + }, + ], + }, + "consumption_drift": { + "report_only": True, + "totals": { + "petrol_spending": { + "design": 100.0, + "final": 102.0, + "change": 2.0, + "relative_change": 0.02, + } + }, + "shares": { + "fuel_car_households_without_fuel": { + "design": 0.0, + "final": 0.0, + "change": 0.0, + "relative_change": None, + } + }, + }, + } + return payload diff --git a/uv.lock b/uv.lock index dc7cbbc95c..1a6ffe763e 100644 --- a/uv.lock +++ b/uv.lock @@ -954,6 +954,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "jsonschema" }, { name = "pytest" }, ] @@ -961,7 +962,10 @@ dev = [ requires-dist = [{ name = "pydantic", specifier = ">=2.10,<3" }] [package.metadata.requires-dev] -dev = [{ name = "pytest", specifier = ">=8" }] +dev = [ + { name = "jsonschema", specifier = ">=4.23,<5" }, + { name = "pytest", specifier = ">=8" }, +] [[package]] name = "microcosm-emitter" From b7cf8a256f00415520097d5f2b23f4b82c337b06 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:53:47 +0400 Subject: [PATCH 2/4] Commit the schema-8 JSON Schema and validate US and UK diagnostics against it Consumers need the schema as a file at a stable path, and the file must not drift from the models. It is committed at packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json, where it ships in the wheel. tools/generate_calibration_diagnostics_json_schema.py regenerates it, or checks it with --check. A byte-exact drift test fails while the file differs from freshly generated output. The engine-free test builders that already write real schema-8 diagnostics now also validate them against the committed file, after strict JSON parsing: - the US national release writer; - the ACS local-area calibrate stage; - the UK writer, with and without skipped targets and consumption drift; - the UK graph terminal gate kernel, with a measured rotated holdout; - the transport toy country's terminal kernel. The UK kernel test and the US NaN-loss test move their setup into helpers so the new tests share it. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 + ...libration-diagnostics-json-schema.added.md | 1 + docs/agent-guide.md | 11 + .../shared/test_transport_terminal_kernels.py | 10 + .../engine_free/uk/test_uk_diagnostics.py | 35 + .../engine_free/uk/test_uk_graph_terminal.py | 37 +- .../test_us_acs_local_target_loss_weights.py | 11 + .../us/test_us_fiscal_refresh_builder.py | 30 +- packages/microcosm-diagnostics/README.md | 11 + .../src/microcosm/diagnostics/__init__.py | 10 + .../src/microcosm/diagnostics/json_schema.py | 72 +- .../calibration_diagnostics.v8.schema.json | 1712 +++++++++++++++++ .../engine_free/shared/test_json_schema.py | 66 + .../microcosm_diagnostics/json_schema.py | 10 + ...ate_calibration_diagnostics_json_schema.py | 43 + 15 files changed, 2052 insertions(+), 14 deletions(-) create mode 100644 changelog.d/1293-calibration-diagnostics-json-schema.added.md create mode 100644 packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json create mode 100644 tools/generate_calibration_diagnostics_json_schema.py diff --git a/README.md b/README.md index 41dd8a2edd..e38a0d0e36 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,13 @@ UK release workflows stop when diagnostics are unavailable because their later release checks require that evidence; historical schema-6 and schema-7 UK artifacts remain readable through isolated compatibility validation. +A Draft 2020-12 JSON Schema for schema 8, generated from those models, is +committed at +`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` +for consumers that check the file without importing them, such as the +collector. Its description lists the model rules a JSON Schema cannot express; +see the [package README](packages/microcosm-diagnostics/README.md). + Standard publication uploads the locally built `releases//` artifacts to the Hugging Face dataset, tags the release, and updates `latest.json`. It runs on the build machine (it needs the freshly built H5), so it isn't a CI step: diff --git a/changelog.d/1293-calibration-diagnostics-json-schema.added.md b/changelog.d/1293-calibration-diagnostics-json-schema.added.md new file mode 100644 index 0000000000..20eff0cd7f --- /dev/null +++ b/changelog.d/1293-calibration-diagnostics-json-schema.added.md @@ -0,0 +1 @@ +The calibration-diagnostics schema-8 JSON Schema is committed at `packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` and ships in the `microcosm-diagnostics` wheel (microcosm#1293). `tools/generate_calibration_diagnostics_json_schema.py` regenerates it, and `--check` reports a stale file. A byte-exact drift test keeps the file equal to freshly generated output. In their engine-free tests, the US national release writer, the ACS local-area calibrate stage, the UK writer, the UK graph terminal gate kernel and the transport terminal kernel now validate the diagnostics they write against the committed file. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index b14084800d..f2bf98b23d 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -411,6 +411,17 @@ label, search for an existing definition and follow AI assistants must import shared static data from its domain-specific constants module instead of copying it or reconstructing alternate views in consumers. +## Calibration diagnostics JSON Schema + +`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` +is generated from the schema-8 models in `microcosm.diagnostics.schema`. After +changing one, regenerate it with +`uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`. +A new model validator also needs an entry in +`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS` and a document in +`test_json_schema.py` that the schema accepts and the models refuse; the +diagnostics tests fail until both exist. + ## Review this guide Update this guide in the same PR whenever the workspace layout, test diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py index 210071681c..8864f859c6 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py @@ -88,6 +88,10 @@ with_config, write_toy_sources, ) +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) DIAGNOSTICS = "toy.calibration.diagnostics" _KEY = "b" * 64 @@ -179,6 +183,12 @@ def test_diagnostics_validate_against_schema_8(toy_run, tmp_path) -> None: ) +def test_transport_diagnostics_match_the_published_json_schema(toy_run) -> None: + payload = toy_run.artifact(DIAGNOSTICS, "diagnostics") + + assert published_json_schema_errors(load_strict_json(payload)) == [] + + def _diagnostics_artifacts(run, **overrides) -> dict[str, ArtifactValue]: artifacts = { alias: _value(run.artifact(producer, name), kind) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py index cb928bd439..af29f53fba 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py @@ -37,6 +37,10 @@ DiagnosticsWriteSuccess, ) from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) _SPI_COLUMN = "household_is_spi_synthetic" _CG_COLUMN = "household_is_capital_gains_clone" @@ -947,6 +951,37 @@ def test_writer_round_trips_strict_json(tmp_path: Path) -> None: assert list(tmp_path.glob(f".{path.name}.*.tmp")) == [] +@pytest.mark.parametrize( + ("with_skipped", "with_consumption"), + [ + pytest.param(False, False, id="scored-targets-only"), + pytest.param(True, True, id="skipped-target-and-consumption-drift"), + ], +) +def test_written_uk_diagnostics_match_the_published_json_schema( + tmp_path: Path, + with_skipped: bool, + with_consumption: bool, +) -> None: + result, frame, registry, geography = _diagnostics_case( + with_skipped=with_skipped, + with_consumption=with_consumption, + ) + path = tmp_path / "calibration_diagnostics.json" + + outcome = write_uk_calibration_diagnostics( + result, + path, + frame, + target_geography_levels=geography, + target_registry=registry, + build={"release_id": "fixture"}, + ) + + assert isinstance(outcome, DiagnosticsWriteSuccess) + assert published_json_schema_errors(load_strict_json(path.read_bytes())) == [] + + @pytest.mark.parametrize( "value", [float("nan"), float("inf"), float("-inf")], diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 3665c977f9..aa4678e2fc 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -21,6 +21,10 @@ from test_support.microcosm_build.uk_hierarchy_fixtures import ( uk_fixture_hierarchy, ) +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) def test_export_drops_native_aliases_and_keeps_identity_keyed_assignment(tmp_path): @@ -788,10 +792,13 @@ def test_both_gate_kernels_are_registered(): assert type(registry.get(UKFullHoldoutKernel.ref)) is UKFullHoldoutKernel -def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( - monkeypatch, -): - import json +def _run_uk_full_gate_kernel(monkeypatch): + """Run the terminal gate kernel on the two-household fixture. + + Rebuilding the diagnostics is forbidden, so a returned document proves + the kernel reused the decoded calibration result. + """ + from dataclasses import replace from types import SimpleNamespace @@ -800,7 +807,6 @@ def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( from microcosm.build.uk_runtime.graph_targets import registry_payload from microcosm.build.uk_runtime.graph_terminal import ( UKFullGateKernel, - decode_full_gate_report, uk_terminal_only_code_hash, ) from microcosm.calibrate import ( @@ -1047,9 +1053,19 @@ def forbidden(*args, **kwargs): artifacts=artifacts, sources={"uk_ladder": "fixture"}, ) - stored = UKFullGateKernel(coverage_engine=object(), engine_identity="fixture").run( + return UKFullGateKernel(coverage_engine=object(), engine_identity="fixture").run( context ) + + +def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( + monkeypatch, +): + import json + + from microcosm.build.uk_runtime.graph_terminal import decode_full_gate_report + + stored = _run_uk_full_gate_kernel(monkeypatch) phase, _ = decode_full_gate_report(stored.artifacts["gate_report"]) assert phase.phase == "terminal" document = json.loads(stored.artifacts["calibration_diagnostics"]) @@ -1064,6 +1080,15 @@ def forbidden(*args, **kwargs): assert support_csv["support_below_floor"].dtype == bool +def test_final_gate_kernel_diagnostics_match_the_published_json_schema( + monkeypatch, +): + stored = _run_uk_full_gate_kernel(monkeypatch) + document = load_strict_json(stored.artifacts["calibration_diagnostics"]) + + assert published_json_schema_errors(document) == [] + + def test_package_validates_materialized_evidence_against_graph_bytes(tmp_path): import json from types import SimpleNamespace diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py index a5e7870310..eaa1a41c4b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py @@ -32,6 +32,10 @@ from test_support.microcosm_build.us_batched_target_materialization import ( _nested_frame, ) +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -384,6 +388,13 @@ def test_every_calibrate_output_records_the_weights(calibrate_run) -> None: ) +def test_calibrate_diagnostics_match_the_published_json_schema(calibrate_run) -> None: + calibrate_run.run() + written = calibrate_run.path / "calibration_diagnostics.json" + + assert published_json_schema_errors(load_strict_json(written.read_bytes())) == [] + + def test_a_family_multiplier_reaches_calibrate_and_the_stamp(calibrate_run) -> None: calls = calibrate_run.run(target_family_loss_multipliers={"usda_snap": 4.0}) train = _train_specs(calibrate_run) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index a1d494147c..77ddcd415f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -27,6 +27,10 @@ ) from microcosm.data.stored_inputs import register_sha256 from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from test_support.microcosm_diagnostics.json_schema import ( + load_strict_json, + published_json_schema_errors, +) from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -5895,10 +5899,9 @@ def fake_write_calibration_diagnostics( } -def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( - small_frame, - tmp_path, -) -> None: +def _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) -> Path: + """Write release diagnostics for a one-target US registry with a NaN loss.""" + builder = _load_builder_module() base_h5 = tmp_path / "base.h5" base_h5.write_bytes(b"h5") @@ -5971,8 +5974,16 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( # writer; keep that call boundary explicit rather than moving the fix. default_dataset={"method": "dense_no_l0", "final_loss": None}, ) + return tmp_path / "calibration_diagnostics.json" + + +def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( + small_frame, + tmp_path, +) -> None: + written = _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) - diagnostics = json.loads((tmp_path / "calibration_diagnostics.json").read_text()) + diagnostics = json.loads(written.read_text()) assert diagnostics["schema_version"] == 8 assert diagnostics["targets"][0]["source"] == "fixture" assert diagnostics["targets"][0]["hierarchy"] == { @@ -5997,6 +6008,15 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( assert diagnostics["build"]["default_dataset"]["final_loss"] is None +def test_release_calibration_diagnostics_match_the_published_json_schema( + small_frame, + tmp_path, +) -> None: + written = _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) + + assert published_json_schema_errors(load_strict_json(written.read_bytes())) == [] + + #: What the stubbed stored-input gate refuses in the ``stored_input_refused`` #: harness mode, in the gate's own line format. STORED_INPUT_REFUSAL_FIXTURE = ( diff --git a/packages/microcosm-diagnostics/README.md b/packages/microcosm-diagnostics/README.md index 7f308cf50a..f7bfeec75d 100644 --- a/packages/microcosm-diagnostics/README.md +++ b/packages/microcosm-diagnostics/README.md @@ -36,3 +36,14 @@ schema's description lists each of these rules (`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS`), and a test fails if a model validator is added without being listed. Complete validation still goes through `parse_calibration_diagnostics`. + +The generated schema is committed, and shipped in the wheel, at + +`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` + +`load_calibration_diagnostics_v8_json_schema()` reads it. After changing a +schema-8 model, regenerate it with +`uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`; +`--check` reports a stale file without writing. A test fails while the +committed file differs from freshly generated output, and the US, UK and +transport build tests validate the diagnostics they write against it. diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py b/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py index 24f58a56f8..cee51f059a 100644 --- a/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py @@ -4,10 +4,15 @@ CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS, CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID, CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE, + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE, JSON_SCHEMA_DRAFT_2020_12, JsonSchemaGap, + assert_calibration_diagnostics_v8_json_schema_current, calibration_diagnostics_v8_json_schema, + committed_calibration_diagnostics_v8_json_schema_path, + load_calibration_diagnostics_v8_json_schema, render_calibration_diagnostics_v8_json_schema, + write_calibration_diagnostics_v8_json_schema, ) from microcosm.diagnostics.schema import ( CALIBRATION_DIAGNOSTICS_ADAPTER, @@ -48,6 +53,7 @@ "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS", "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID", "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE", + "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE", "JSON_SCHEMA_DRAFT_2020_12", "SUPPORTED_CALIBRATION_DIAGNOSTICS_SCHEMA_VERSIONS", "CalibrationDiagnostics", @@ -72,12 +78,16 @@ "UKWeakestFamily", "UKWeightSummary", "UKZeroWeightStratum", + "assert_calibration_diagnostics_v8_json_schema_current", "calibration_diagnostics_json_schema", "calibration_diagnostics_v8_json_schema", + "committed_calibration_diagnostics_v8_json_schema_path", "failed_diagnostics_outcome", + "load_calibration_diagnostics_v8_json_schema", "parse_calibration_diagnostics", "render_calibration_diagnostics_v8_json_schema", "write_calibration_diagnostics", + "write_calibration_diagnostics_v8_json_schema", ] __version__ = "0.1.0" diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py b/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py index 761612ef77..78cdb76d61 100644 --- a/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/json_schema.py @@ -11,6 +11,8 @@ import json from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path from typing import Final from pydantic import JsonValue @@ -21,6 +23,13 @@ CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID: Final = ( "calibration_diagnostics.v8.schema.json" ) +#: The committed schema, relative to the ``microcosm.diagnostics`` package. +CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE: Final = ( + f"schemas/{CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID}" +) +_REGENERATE_COMMAND: Final = ( + "uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py" +) @dataclass(frozen=True) @@ -226,9 +235,8 @@ class JsonSchemaGap: _TITLE: Final = "Microcosm calibration diagnostics, schema 8" _COMMENT: Final = ( - "Generated from microcosm.diagnostics.CalibrationDiagnosticsV8 by " - "microcosm.diagnostics.calibration_diagnostics_v8_json_schema(); " - "regenerate it rather than editing it." + "Generated from microcosm.diagnostics.CalibrationDiagnosticsV8. Do not " + f"edit it; regenerate it with `{_REGENERATE_COMMAND}`." ) @@ -279,3 +287,61 @@ def render_calibration_diagnostics_v8_json_schema() -> str: ) + "\n" ) + + +def committed_calibration_diagnostics_v8_json_schema_path() -> Path: + """Return the committed schema file, shipped inside the package.""" + + return Path( + str( + files("microcosm.diagnostics").joinpath( + CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE + ) + ) + ) + + +def load_calibration_diagnostics_v8_json_schema( + path: str | Path | None = None, +) -> dict[str, JsonValue]: + """Read the committed schema, or the schema file at ``path``.""" + + source = ( + committed_calibration_diagnostics_v8_json_schema_path() + if path is None + else Path(path) + ) + return json.loads(source.read_bytes()) + + +def write_calibration_diagnostics_v8_json_schema( + path: str | Path | None = None, +) -> Path: + """Write freshly generated schema text to the committed file or ``path``.""" + + output = ( + committed_calibration_diagnostics_v8_json_schema_path() + if path is None + else Path(path) + ) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(render_calibration_diagnostics_v8_json_schema().encode("utf-8")) + return output + + +def assert_calibration_diagnostics_v8_json_schema_current( + path: str | Path | None = None, +) -> None: + """Raise unless the file is byte-identical to freshly generated output.""" + + source = ( + committed_calibration_diagnostics_v8_json_schema_path() + if path is None + else Path(path) + ) + expected = render_calibration_diagnostics_v8_json_schema().encode("utf-8") + if source.read_bytes() != expected: + raise ValueError( + f"{source} differs from the schema the models generate; " + f"regenerate it with `{_REGENERATE_COMMAND}`." + ) diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json new file mode 100644 index 0000000000..6fb0660440 --- /dev/null +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json @@ -0,0 +1,1712 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "calibration_diagnostics.v8.schema.json", + "$comment": "Generated from microcosm.diagnostics.CalibrationDiagnosticsV8. Do not edit it; regenerate it with `uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`.", + "title": "Microcosm calibration diagnostics, schema 8", + "description": "The calibration_diagnostics.json document, schema 8, that Microcosm builds write beside a calibrated dataset. The pydantic models it is generated from remain the authority. A document that satisfies this schema can still break one of the rules below, which the models enforce and this schema does not express, so validate with microcosm.diagnostics.parse_calibration_diagnostics for the complete contract. Equalities between floating-point values hold within the models' tolerances.\n\nRules the models enforce that this schema does not express:\n- Every number is finite, at any depth, including inside options, targets[].metadata and build. JSON has no NaN or Infinity, but lenient parsers such as Python's json module accept those tokens, and a JSON Schema number cannot exclude the values they produce, so parse submissions with a parser that refuses them.\n- Integer fields hold integers written without a fraction or exponent. JSON Schema counts 2.0 and 1e3 as integers; the strict models refuse them, because Python's json module decodes them as floats.\n- Strings the models type as non-blank contain at least one character that is not whitespace. These are most names, identifiers, labels, sources, entities, reasons, codes and messages, and the keys of uk_diagnostics.target_observation_basis, the zero-weight strata and the consumption-drift maps.\n- Digests match ^[0-9a-f]{64}$ with nothing after them. In the ECMA-262 regular expressions JSON Schema specifies, $ matches only at the end of the string, but validators that evaluate pattern with Python's re module also let $ match before a final newline, so they accept a digest followed by a newline.\n- In each targets[].hierarchy, category.provider_id equals provider.id and the dimensions have unique ids.\n- Each targets[].hierarchy.target.id equals that row's target_name.\n- target_surface.constraint_matrix has n_targets rows and n_records columns, and its nnz is at most rows times columns.\n- target_surface.n_targets equals the number of targets, target_registry.n_specs is at least the number of targets, target_surface.n_records equals n_records, and n_nonzero is at most n_records.\n- A UK registry (target_registry.country \"uk\") requires uk_diagnostics; any other registry requires it to be null or absent. In a UK document, uk_diagnostics.weights.n_records, effective_sample_size and top_1pct_weight_share equal the top-level n_records, effective_sample_size and top_1pct_weight_share. The n_targets, n_scored and n_skipped values in uk_diagnostics.target_pass_rates_by_geography_level sum to target_registry.n_specs, the number of targets and the number of skipped targets.\n- In uk_diagnostics.weights, positive_weight_records plus zero_weight_records equals n_records, effective_sample_size is at most n_records, and ess_fraction equals effective_sample_size divided by n_records. median_positive_weight and max_to_median_positive_weight are null exactly when positive_weight_records is 0; otherwise max_to_median_positive_weight equals max_weight divided by median_positive_weight.\n- In each uk_diagnostics.zero_weight_rows_by_stratum row, positive_weight_rows plus zero_weight_rows equals rows. Across the strata, rows, positive_weight_rows and zero_weight_rows sum to weights.n_records, weights.positive_weight_records and weights.zero_weight_records.\n- uk_diagnostics.target_pass_rates_by_geography_level has exactly one row for each of national, region, country, local_authority and constituency. In each row, n_scored plus n_skipped equals n_targets, n_within_10pct is at most n_scored, and pass_rate equals n_within_10pct divided by n_targets, or is null when n_targets is 0.\n- uk_diagnostics.weakest_families and weakest_areas_by_fit are both present or both absent. In weakest_families, weakest_areas_by_fit.bottom_by_fit and weakest_areas_by_fit.countries, n_within_10pct is at most n_targets and pass_rate equals n_within_10pct divided by n_targets. bottom_by_fit has at most limit rows and at most n_areas_scored rows.\n- A measured uk_diagnostics.rotated_holdout has unique grains and n_folds entries in both fold_losses and folds. Its fold ids cover 0 to n_folds - 1, each fold's holdout_loss equals its fold_losses entry, mean_holdout_loss is the mean of fold_losses and worst_holdout_loss is their maximum. Each fold has n_holdout_targets unique holdout_target_indices.\n- Each uk_diagnostics.consumption_drift value has change equal to final minus design, and its relative_change is null exactly when design is 0.\n\nWhere this schema is stricter than the models' parser:\n- Literal fields hold exactly their declared JSON value. The models compare literals with Python equality, so their parser also accepts 1 for true and true for 1, for example in rotated_holdout.skipped or uk_diagnostics.schema_version. The canonical writer always emits the declared value, which this schema requires.", + "$defs": { + "CalibrationHierarchy": { + "additionalProperties": false, + "properties": { + "provider": { + "$ref": "#/$defs/HierarchyNode" + }, + "category": { + "$ref": "#/$defs/HierarchyCategory" + }, + "geography": { + "$ref": "#/$defs/HierarchyGeography" + }, + "dimensions": { + "items": { + "$ref": "#/$defs/HierarchyDimension" + }, + "title": "Dimensions", + "type": "array" + }, + "target": { + "$ref": "#/$defs/HierarchyNode" + } + }, + "required": [ + "provider", + "category", + "geography", + "dimensions", + "target" + ], + "title": "CalibrationHierarchy", + "type": "object" + }, + "DiagnosticsWarning": { + "additionalProperties": false, + "properties": { + "code": { + "title": "Code", + "type": "string" + }, + "severity": { + "const": "warning", + "title": "Severity", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + } + }, + "required": [ + "code", + "severity", + "message" + ], + "title": "DiagnosticsWarning", + "type": "object" + }, + "HierarchyCategory": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "provider_id": { + "title": "Provider Id", + "type": "string" + } + }, + "required": [ + "id", + "label", + "provider_id" + ], + "title": "HierarchyCategory", + "type": "object" + }, + "HierarchyDimension": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "value_id": { + "title": "Value Id", + "type": "string" + }, + "value_label": { + "title": "Value Label", + "type": "string" + } + }, + "required": [ + "id", + "label", + "value_id", + "value_label" + ], + "title": "HierarchyDimension", + "type": "object" + }, + "HierarchyGeography": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + }, + "level": { + "title": "Level", + "type": "string" + } + }, + "required": [ + "id", + "label", + "level" + ], + "title": "HierarchyGeography", + "type": "object" + }, + "HierarchyNode": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "id", + "label" + ], + "title": "HierarchyNode", + "type": "object" + }, + "JsonValue": {}, + "PastCapCensus": { + "additionalProperties": false, + "properties": { + "cap": { + "exclusiveMinimum": 0, + "title": "Cap", + "type": "number" + }, + "scale_basis": { + "title": "Scale Basis", + "type": "string" + }, + "n_targets": { + "minimum": 0, + "title": "N Targets", + "type": "integer" + }, + "initial_past_cap": { + "minimum": 0, + "title": "Initial Past Cap", + "type": "integer" + }, + "final_past_cap": { + "minimum": 0, + "title": "Final Past Cap", + "type": "integer" + }, + "escaped": { + "minimum": 0, + "title": "Escaped", + "type": "integer" + }, + "frozen": { + "minimum": 0, + "title": "Frozen", + "type": "integer" + }, + "pushed_out": { + "minimum": 0, + "title": "Pushed Out", + "type": "integer" + }, + "pushed_out_rows": { + "items": { + "$ref": "#/$defs/PushedOutTarget" + }, + "title": "Pushed Out Rows", + "type": "array" + } + }, + "required": [ + "cap", + "scale_basis", + "n_targets", + "initial_past_cap", + "final_past_cap", + "escaped", + "frozen", + "pushed_out", + "pushed_out_rows" + ], + "title": "PastCapCensus", + "type": "object" + }, + "PushedOutTarget": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "init_rel": { + "title": "Init Rel", + "type": "number" + }, + "final_rel": { + "title": "Final Rel", + "type": "number" + } + }, + "required": [ + "name", + "init_rel", + "final_rel" + ], + "title": "PushedOutTarget", + "type": "object" + }, + "RegistryTargetMetadata": { + "additionalProperties": false, + "properties": { + "family": { + "title": "Family", + "type": "string" + }, + "se": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Se" + }, + "signed": { + "title": "Signed", + "type": "boolean" + }, + "notes": { + "title": "Notes", + "type": "string" + } + }, + "required": [ + "family", + "se", + "signed", + "notes" + ], + "title": "RegistryTargetMetadata", + "type": "object" + }, + "Selector": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "column", + "callable" + ], + "title": "Kind", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "kind", + "name" + ], + "title": "Selector", + "type": "object" + }, + "SkippedTarget": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "reason": { + "title": "Reason", + "type": "string" + } + }, + "required": [ + "name", + "reason" + ], + "title": "SkippedTarget", + "type": "object" + }, + "TargetDiagnosticV8": { + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "target_name": { + "title": "Target Name", + "type": "string" + }, + "period": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "title": "Period" + }, + "entity": { + "title": "Entity", + "type": "string" + }, + "measure": { + "$ref": "#/$defs/Selector" + }, + "filter": { + "anyOf": [ + { + "$ref": "#/$defs/Selector" + }, + { + "type": "null" + } + ] + }, + "source": { + "title": "Source", + "type": "string" + }, + "metadata": { + "additionalProperties": { + "$ref": "#/$defs/JsonValue" + }, + "title": "Metadata", + "type": "object" + }, + "target": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Target" + }, + "compiled_target": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Compiled Target" + }, + "initial_estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Initial Estimate" + }, + "final_estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Final Estimate" + }, + "relative_error": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Error" + }, + "within_tolerance": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Within Tolerance" + }, + "hierarchy": { + "$ref": "#/$defs/CalibrationHierarchy" + }, + "registry": { + "anyOf": [ + { + "$ref": "#/$defs/RegistryTargetMetadata" + }, + { + "type": "null" + } + ], + "default": null + }, + "target_loss_weight": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Loss Weight" + }, + "target_loss_weight_share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Loss Weight Share" + }, + "target_loss_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target Loss Scale" + }, + "final_capped_scaled_error": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Final Capped Scaled Error" + }, + "final_loss_contribution": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Final Loss Contribution" + } + }, + "required": [ + "name", + "target_name", + "period", + "entity", + "measure", + "filter", + "source", + "metadata", + "target", + "compiled_target", + "initial_estimate", + "final_estimate", + "relative_error", + "within_tolerance", + "hierarchy" + ], + "title": "TargetDiagnosticV8", + "type": "object" + }, + "TargetLossBasis": { + "additionalProperties": false, + "properties": { + "formula": { + "title": "Formula", + "type": "string" + }, + "cap": { + "exclusiveMinimum": 0, + "title": "Cap", + "type": "number" + }, + "target_count": { + "minimum": 0, + "title": "Target Count", + "type": "integer" + }, + "total_target_weight": { + "exclusiveMinimum": 0, + "title": "Total Target Weight", + "type": "number" + }, + "weight_kind": { + "title": "Weight Kind", + "type": "string" + }, + "scale_kind": { + "title": "Scale Kind", + "type": "string" + }, + "hash_algorithm": { + "title": "Hash Algorithm", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "formula", + "cap", + "target_count", + "total_target_weight", + "weight_kind", + "scale_kind", + "hash_algorithm", + "sha256" + ], + "title": "TargetLossBasis", + "type": "object" + }, + "TargetRegistryRef": { + "additionalProperties": false, + "properties": { + "country": { + "title": "Country", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + }, + "n_specs": { + "exclusiveMinimum": 0, + "title": "N Specs", + "type": "integer" + } + }, + "required": [ + "country", + "version", + "n_specs" + ], + "title": "TargetRegistryRef", + "type": "object" + }, + "TargetSurface": { + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "weight_entity": { + "title": "Weight Entity", + "type": "string" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_records": { + "exclusiveMinimum": 0, + "title": "N Records", + "type": "integer" + }, + "constraint_matrix": { + "$ref": "#/$defs/TargetSurfaceMatrix" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "names_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Names Sha256", + "type": "string" + }, + "values_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Values Sha256", + "type": "string" + } + }, + "required": [ + "schema_version", + "weight_entity", + "n_targets", + "n_records", + "constraint_matrix", + "sha256", + "names_sha256", + "values_sha256" + ], + "title": "TargetSurface", + "type": "object" + }, + "TargetSurfaceMatrix": { + "additionalProperties": false, + "properties": { + "rows": { + "minimum": 0, + "title": "Rows", + "type": "integer" + }, + "columns": { + "minimum": 0, + "title": "Columns", + "type": "integer" + }, + "nnz": { + "minimum": 0, + "title": "Nnz", + "type": "integer" + } + }, + "required": [ + "rows", + "columns", + "nnz" + ], + "title": "TargetSurfaceMatrix", + "type": "object" + }, + "UKAreaFit": { + "additionalProperties": false, + "properties": { + "geography_level": { + "enum": [ + "local_authority", + "constituency" + ], + "title": "Geography Level", + "type": "string" + }, + "area_code": { + "title": "Area Code", + "type": "string" + }, + "country": { + "enum": [ + "England", + "Northern Ireland", + "Scotland", + "Wales" + ], + "title": "Country", + "type": "string" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "maximum": 1, + "minimum": 0, + "title": "Pass Rate", + "type": "number" + }, + "worst_target": { + "title": "Worst Target", + "type": "string" + }, + "worst_abs_relative_error": { + "minimum": 0, + "title": "Worst Abs Relative Error", + "type": "number" + }, + "loss_contribution": { + "minimum": 0, + "title": "Loss Contribution", + "type": "number" + }, + "nonzero_households": { + "minimum": 0, + "title": "Nonzero Households", + "type": "integer" + }, + "nonzero_source_households": { + "minimum": 0, + "title": "Nonzero Source Households", + "type": "integer" + }, + "effective_sample_size": { + "minimum": 0, + "title": "Effective Sample Size", + "type": "number" + } + }, + "required": [ + "geography_level", + "area_code", + "country", + "n_targets", + "n_within_10pct", + "pass_rate", + "worst_target", + "worst_abs_relative_error", + "loss_contribution", + "nonzero_households", + "nonzero_source_households", + "effective_sample_size" + ], + "title": "UKAreaFit", + "type": "object" + }, + "UKConsumptionDrift": { + "additionalProperties": false, + "description": "Stage-levelled spend and its composition, design against final weights.\n\nReport only (microcosm#1113): the stage sets these levels and shares, so\ncalibration should barely move them; a large move means a bound row is\npulling weight onto the columns instead.", + "properties": { + "report_only": { + "const": true, + "title": "Report Only", + "type": "boolean" + }, + "totals": { + "additionalProperties": { + "$ref": "#/$defs/UKDriftValue" + }, + "minProperties": 1, + "title": "Totals", + "type": "object" + }, + "shares": { + "additionalProperties": { + "$ref": "#/$defs/UKDriftValue" + }, + "minProperties": 1, + "title": "Shares", + "type": "object" + } + }, + "required": [ + "report_only", + "totals", + "shares" + ], + "title": "UKConsumptionDrift", + "type": "object" + }, + "UKCountryFit": { + "additionalProperties": false, + "properties": { + "country": { + "enum": [ + "England", + "Northern Ireland", + "Scotland", + "Wales" + ], + "title": "Country", + "type": "string" + }, + "geography_level": { + "enum": [ + "local_authority", + "constituency" + ], + "title": "Geography Level", + "type": "string" + }, + "n_areas": { + "exclusiveMinimum": 0, + "title": "N Areas", + "type": "integer" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "maximum": 1, + "minimum": 0, + "title": "Pass Rate", + "type": "number" + }, + "worst_target": { + "title": "Worst Target", + "type": "string" + }, + "worst_abs_relative_error": { + "minimum": 0, + "title": "Worst Abs Relative Error", + "type": "number" + }, + "loss_contribution": { + "minimum": 0, + "title": "Loss Contribution", + "type": "number" + } + }, + "required": [ + "country", + "geography_level", + "n_areas", + "n_targets", + "n_within_10pct", + "pass_rate", + "worst_target", + "worst_abs_relative_error", + "loss_contribution" + ], + "title": "UKCountryFit", + "type": "object" + }, + "UKDiagnosticsV1": { + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1, + "title": "Schema Version", + "type": "integer" + }, + "weights": { + "$ref": "#/$defs/UKWeightSummary" + }, + "zero_weight_rows_by_stratum": { + "items": { + "$ref": "#/$defs/UKZeroWeightStratum" + }, + "minItems": 1, + "title": "Zero Weight Rows By Stratum", + "type": "array" + }, + "target_pass_rates_by_geography_level": { + "items": { + "$ref": "#/$defs/UKGeographyPassRate" + }, + "title": "Target Pass Rates By Geography Level", + "type": "array" + }, + "target_observation_basis": { + "additionalProperties": { + "type": "string" + }, + "title": "Target Observation Basis", + "type": "object" + }, + "weakest_families": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/UKWeakestFamily" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Weakest Families" + }, + "weakest_areas_by_fit": { + "anyOf": [ + { + "$ref": "#/$defs/UKWeakestAreasByFit" + }, + { + "type": "null" + } + ], + "default": null + }, + "rotated_holdout": { + "anyOf": [ + { + "$ref": "#/$defs/UKMeasuredRotatedHoldout" + }, + { + "$ref": "#/$defs/UKSkippedRotatedHoldout" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rotated Holdout" + }, + "consumption_drift": { + "anyOf": [ + { + "$ref": "#/$defs/UKConsumptionDrift" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "schema_version", + "weights", + "zero_weight_rows_by_stratum", + "target_pass_rates_by_geography_level", + "target_observation_basis" + ], + "title": "UKDiagnosticsV1", + "type": "object" + }, + "UKDriftValue": { + "additionalProperties": false, + "description": "One quantity at the design weights and at the final weights.", + "properties": { + "design": { + "title": "Design", + "type": "number" + }, + "final": { + "title": "Final", + "type": "number" + }, + "change": { + "title": "Change", + "type": "number" + }, + "relative_change": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Change" + } + }, + "required": [ + "design", + "final", + "change", + "relative_change" + ], + "title": "UKDriftValue", + "type": "object" + }, + "UKGeographyPassRate": { + "additionalProperties": false, + "properties": { + "geography_level": { + "enum": [ + "national", + "region", + "country", + "local_authority", + "constituency" + ], + "title": "Geography Level", + "type": "string" + }, + "n_targets": { + "minimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_scored": { + "minimum": 0, + "title": "N Scored", + "type": "integer" + }, + "n_skipped": { + "minimum": 0, + "title": "N Skipped", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Pass Rate" + } + }, + "required": [ + "geography_level", + "n_targets", + "n_scored", + "n_skipped", + "n_within_10pct", + "pass_rate" + ], + "title": "UKGeographyPassRate", + "type": "object" + }, + "UKMeasuredRotatedHoldout": { + "additionalProperties": false, + "properties": { + "report_only": { + "const": true, + "title": "Report Only", + "type": "boolean" + }, + "method": { + "const": "rotated_folds", + "title": "Method", + "type": "string" + }, + "target_loss_cap": { + "exclusiveMinimum": 0, + "title": "Target Loss Cap", + "type": "number" + }, + "loss_weight_scale": { + "const": "held_local_grains_only", + "title": "Loss Weight Scale", + "type": "string" + }, + "target_weight_rule": { + "enum": [ + "uniform", + "grain_equal", + "grain_family_equal", + "grain_family_equal_sqrt_count", + "nation_grain_family_equal", + "nation_grain_family_equal_sqrt_count" + ], + "title": "Target Weight Rule", + "type": "string" + }, + "population": { + "const": "held_out_local_targets", + "title": "Population", + "type": "string" + }, + "grains": { + "items": { + "enum": [ + "constituency", + "local_authority", + "la" + ], + "type": "string" + }, + "minItems": 1, + "title": "Grains", + "type": "array" + }, + "n_folds": { + "minimum": 2, + "title": "N Folds", + "type": "integer" + }, + "seed": { + "title": "Seed", + "type": "integer" + }, + "solve_seed": { + "title": "Solve Seed", + "type": "integer" + }, + "mean_holdout_loss": { + "minimum": 0, + "title": "Mean Holdout Loss", + "type": "number" + }, + "worst_holdout_loss": { + "minimum": 0, + "title": "Worst Holdout Loss", + "type": "number" + }, + "fold_losses": { + "items": { + "minimum": 0, + "type": "number" + }, + "title": "Fold Losses", + "type": "array" + }, + "folds": { + "items": { + "$ref": "#/$defs/UKRotatedHoldoutFold" + }, + "title": "Folds", + "type": "array" + } + }, + "required": [ + "report_only", + "method", + "target_loss_cap", + "loss_weight_scale", + "target_weight_rule", + "population", + "grains", + "n_folds", + "seed", + "solve_seed", + "mean_holdout_loss", + "worst_holdout_loss", + "fold_losses", + "folds" + ], + "title": "UKMeasuredRotatedHoldout", + "type": "object" + }, + "UKRotatedHoldoutFold": { + "additionalProperties": false, + "properties": { + "fold": { + "minimum": 0, + "title": "Fold", + "type": "integer" + }, + "n_train_targets": { + "minimum": 0, + "title": "N Train Targets", + "type": "integer" + }, + "n_holdout_targets": { + "exclusiveMinimum": 0, + "title": "N Holdout Targets", + "type": "integer" + }, + "holdout_target_indices": { + "items": { + "minimum": 0, + "type": "integer" + }, + "title": "Holdout Target Indices", + "type": "array" + }, + "training_national_rows": { + "minimum": 0, + "title": "Training National Rows", + "type": "integer" + }, + "holdout_loss": { + "minimum": 0, + "title": "Holdout Loss", + "type": "number" + } + }, + "required": [ + "fold", + "n_train_targets", + "n_holdout_targets", + "holdout_target_indices", + "training_national_rows", + "holdout_loss" + ], + "title": "UKRotatedHoldoutFold", + "type": "object" + }, + "UKSkippedRotatedHoldout": { + "additionalProperties": false, + "properties": { + "skipped": { + "const": true, + "title": "Skipped", + "type": "boolean" + } + }, + "required": [ + "skipped" + ], + "title": "UKSkippedRotatedHoldout", + "type": "object" + }, + "UKWeakestAreasByFit": { + "additionalProperties": false, + "properties": { + "limit": { + "exclusiveMinimum": 0, + "title": "Limit", + "type": "integer" + }, + "n_areas_scored": { + "minimum": 0, + "title": "N Areas Scored", + "type": "integer" + }, + "bottom_by_fit": { + "items": { + "$ref": "#/$defs/UKAreaFit" + }, + "title": "Bottom By Fit", + "type": "array" + }, + "countries": { + "items": { + "$ref": "#/$defs/UKCountryFit" + }, + "title": "Countries", + "type": "array" + } + }, + "required": [ + "limit", + "n_areas_scored", + "bottom_by_fit", + "countries" + ], + "title": "UKWeakestAreasByFit", + "type": "object" + }, + "UKWeakestFamily": { + "additionalProperties": false, + "properties": { + "family": { + "title": "Family", + "type": "string" + }, + "n_targets": { + "exclusiveMinimum": 0, + "title": "N Targets", + "type": "integer" + }, + "n_within_10pct": { + "minimum": 0, + "title": "N Within 10Pct", + "type": "integer" + }, + "pass_rate": { + "maximum": 1, + "minimum": 0, + "title": "Pass Rate", + "type": "number" + }, + "worst_target": { + "title": "Worst Target", + "type": "string" + }, + "worst_abs_relative_error": { + "minimum": 0, + "title": "Worst Abs Relative Error", + "type": "number" + }, + "loss_contribution": { + "minimum": 0, + "title": "Loss Contribution", + "type": "number" + }, + "loss_share": { + "maximum": 1, + "minimum": 0, + "title": "Loss Share", + "type": "number" + } + }, + "required": [ + "family", + "n_targets", + "n_within_10pct", + "pass_rate", + "worst_target", + "worst_abs_relative_error", + "loss_contribution", + "loss_share" + ], + "title": "UKWeakestFamily", + "type": "object" + }, + "UKWeightSummary": { + "additionalProperties": false, + "properties": { + "n_records": { + "exclusiveMinimum": 0, + "title": "N Records", + "type": "integer" + }, + "positive_weight_records": { + "minimum": 0, + "title": "Positive Weight Records", + "type": "integer" + }, + "zero_weight_records": { + "minimum": 0, + "title": "Zero Weight Records", + "type": "integer" + }, + "total_weight": { + "minimum": 0, + "title": "Total Weight", + "type": "number" + }, + "effective_sample_size": { + "minimum": 0, + "title": "Effective Sample Size", + "type": "number" + }, + "ess_fraction": { + "maximum": 1, + "minimum": 0, + "title": "Ess Fraction", + "type": "number" + }, + "median_positive_weight": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Median Positive Weight" + }, + "max_weight": { + "minimum": 0, + "title": "Max Weight", + "type": "number" + }, + "max_to_median_positive_weight": { + "anyOf": [ + { + "minimum": 1, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Max To Median Positive Weight" + }, + "top_1pct_weight_share": { + "maximum": 1, + "minimum": 0, + "title": "Top 1Pct Weight Share", + "type": "number" + } + }, + "required": [ + "n_records", + "positive_weight_records", + "zero_weight_records", + "total_weight", + "effective_sample_size", + "ess_fraction", + "median_positive_weight", + "max_weight", + "max_to_median_positive_weight", + "top_1pct_weight_share" + ], + "title": "UKWeightSummary", + "type": "object" + }, + "UKZeroWeightStratum": { + "additionalProperties": false, + "properties": { + "stratum": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "minProperties": 1, + "title": "Stratum", + "type": "object" + }, + "rows": { + "minimum": 0, + "title": "Rows", + "type": "integer" + }, + "positive_weight_rows": { + "minimum": 0, + "title": "Positive Weight Rows", + "type": "integer" + }, + "zero_weight_rows": { + "minimum": 0, + "title": "Zero Weight Rows", + "type": "integer" + }, + "weight_sum": { + "minimum": 0, + "title": "Weight Sum", + "type": "number" + } + }, + "required": [ + "stratum", + "rows", + "positive_weight_rows", + "zero_weight_rows", + "weight_sum" + ], + "title": "UKZeroWeightStratum", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 8, + "title": "Schema Version", + "type": "integer" + }, + "weight_entity": { + "title": "Weight Entity", + "type": "string" + }, + "options": { + "additionalProperties": { + "$ref": "#/$defs/JsonValue" + }, + "title": "Options", + "type": "object" + }, + "target_surface": { + "$ref": "#/$defs/TargetSurface" + }, + "target_registry": { + "$ref": "#/$defs/TargetRegistryRef" + }, + "l0_lambda": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "L0 Lambda" + }, + "n_nonzero": { + "minimum": 0, + "title": "N Nonzero", + "type": "integer" + }, + "n_records": { + "exclusiveMinimum": 0, + "title": "N Records", + "type": "integer" + }, + "initial_loss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Initial Loss" + }, + "final_loss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Final Loss" + }, + "fraction_within_10pct": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Fraction Within 10Pct" + }, + "effective_sample_size": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Effective Sample Size" + }, + "realized_max_weight_ratio": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Realized Max Weight Ratio" + }, + "top_1pct_weight_share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Top 1Pct Weight Share" + }, + "loss_trajectory": { + "items": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "Loss Trajectory", + "type": "array" + }, + "skipped": { + "items": { + "$ref": "#/$defs/SkippedTarget" + }, + "title": "Skipped", + "type": "array" + }, + "past_cap_census": { + "anyOf": [ + { + "$ref": "#/$defs/PastCapCensus" + }, + { + "type": "null" + } + ] + }, + "diagnostic_warnings": { + "items": { + "$ref": "#/$defs/DiagnosticsWarning" + }, + "title": "Diagnostic Warnings", + "type": "array" + }, + "targets": { + "items": { + "$ref": "#/$defs/TargetDiagnosticV8" + }, + "title": "Targets", + "type": "array" + }, + "target_loss_basis": { + "anyOf": [ + { + "$ref": "#/$defs/TargetLossBasis" + }, + { + "type": "null" + } + ], + "default": null + }, + "build": { + "anyOf": [ + { + "additionalProperties": { + "$ref": "#/$defs/JsonValue" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Build" + }, + "uk_diagnostics": { + "anyOf": [ + { + "$ref": "#/$defs/UKDiagnosticsV1" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "schema_version", + "weight_entity", + "options", + "target_surface", + "target_registry", + "l0_lambda", + "n_nonzero", + "n_records", + "initial_loss", + "final_loss", + "fraction_within_10pct", + "effective_sample_size", + "realized_max_weight_ratio", + "top_1pct_weight_share", + "loss_trajectory", + "skipped", + "past_cap_census", + "diagnostic_warnings", + "targets" + ], + "type": "object" +} diff --git a/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py index abb040d1a8..5c58093267 100644 --- a/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py +++ b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py @@ -7,9 +7,11 @@ from __future__ import annotations +import importlib.util import json from collections.abc import Callable from functools import cache +from pathlib import Path import pytest from jsonschema import Draft202012Validator @@ -24,8 +26,15 @@ schema_8_payload, uk_schema_8_payload, ) +from test_support.paths import paths_for +_TEST_PATHS = paths_for("microcosm-diagnostics") DRAFT_2020_12 = "https://json-schema.org/draft/2020-12/schema" +#: The documented location consumers such as the collector read. +PUBLISHED_SCHEMA_PATH = ( + "packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/" + "calibration_diagnostics.v8.schema.json" +) DocumentBuilder = Callable[[], object] DocumentMutation = Callable[[dict], None] @@ -617,3 +626,60 @@ def test_rendered_schema_is_deterministic_json_text() -> None: "title", "description", ] + + +def _load_generator_tool(): + path = ( + _TEST_PATHS.repository + / "tools" + / "generate_calibration_diagnostics_json_schema.py" + ) + spec = importlib.util.spec_from_file_location( + "generate_calibration_diagnostics_json_schema", path + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_committed_schema_matches_the_generated_schema() -> None: + committed = diagnostics.committed_calibration_diagnostics_v8_json_schema_path() + + assert committed.read_bytes() == ( + diagnostics.render_calibration_diagnostics_v8_json_schema().encode("utf-8") + ) + assert diagnostics.load_calibration_diagnostics_v8_json_schema() == ( + diagnostics.calibration_diagnostics_v8_json_schema() + ) + diagnostics.assert_calibration_diagnostics_v8_json_schema_current() + + +def test_committed_schema_lives_at_its_documented_path() -> None: + committed = diagnostics.committed_calibration_diagnostics_v8_json_schema_path() + package_readme = (_TEST_PATHS.package / "README.md").read_text(encoding="utf-8") + + assert committed.resolve() == ( + (_TEST_PATHS.repository / PUBLISHED_SCHEMA_PATH).resolve() + ) + assert committed.name == diagnostics.CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID + assert f"`{PUBLISHED_SCHEMA_PATH}`" in package_readme + + +def test_generator_tool_writes_and_checks_the_committed_schema( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + tool = _load_generator_tool() + committed = diagnostics.committed_calibration_diagnostics_v8_json_schema_path() + written = tmp_path / committed.name + + assert tool.main(["--check"]) == 0 + assert tool.main(["--out", str(written)]) == 0 + assert written.read_bytes() == committed.read_bytes() + + written.write_text("{}\n", encoding="utf-8") + assert tool.main(["--check", "--out", str(written)]) == 1 + assert "regenerate it with" in capsys.readouterr().err + with pytest.raises(ValueError, match="differs from the schema the models generate"): + diagnostics.assert_calibration_diagnostics_v8_json_schema_current(written) diff --git a/test_support/microcosm_diagnostics/json_schema.py b/test_support/microcosm_diagnostics/json_schema.py index 062827e848..307dd193cf 100644 --- a/test_support/microcosm_diagnostics/json_schema.py +++ b/test_support/microcosm_diagnostics/json_schema.py @@ -7,6 +7,8 @@ from jsonschema import Draft202012Validator, ValidationError +from microcosm import diagnostics + def _refuse_non_finite_literal(literal: str) -> float: raise ValueError(f"strict JSON has no {literal} literal") @@ -36,3 +38,11 @@ def json_schema_errors(document: object, schema: Mapping[str, object]) -> list[s f"{error.json_path}: {error.message}" for error in _leaf_errors(validator.iter_errors(document)) ) + + +def published_json_schema_errors(document: object) -> list[str]: + """Return the errors against the committed schema-8 file consumers read.""" + + return json_schema_errors( + document, diagnostics.load_calibration_diagnostics_v8_json_schema() + ) diff --git a/tools/generate_calibration_diagnostics_json_schema.py b/tools/generate_calibration_diagnostics_json_schema.py new file mode 100644 index 0000000000..cd44c9ef71 --- /dev/null +++ b/tools/generate_calibration_diagnostics_json_schema.py @@ -0,0 +1,43 @@ +"""Regenerate or check the committed calibration-diagnostics schema-8 JSON Schema.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from microcosm.diagnostics import ( + assert_calibration_diagnostics_v8_json_schema_current, + committed_calibration_diagnostics_v8_json_schema_path, + write_calibration_diagnostics_v8_json_schema, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--out", + type=Path, + help="write or check this file instead of the committed schema", + ) + parser.add_argument( + "--check", + action="store_true", + help="exit 1 unless the file equals freshly generated output", + ) + args = parser.parse_args(argv) + path = args.out or committed_calibration_diagnostics_v8_json_schema_path() + if args.check: + try: + assert_calibration_diagnostics_v8_json_schema_current(path) + except (OSError, ValueError) as error: + print(f"stale: {error}", file=sys.stderr) + return 1 + print(f"current: {path}") + return 0 + print(write_calibration_diagnostics_v8_json_schema(path)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From c0549d21bbd2e290f84b23e12b95923ab8d13029 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:34:45 +0400 Subject: [PATCH 3/4] Regenerate the schema-8 JSON Schema for UKAreaFit's withheld support figures microcosm#1301 changed UKAreaFit on main: released diagnostics withhold an area's support figures (nonzero_households, nonzero_source_households, effective_sample_size) and carry support_below_floor instead. The figure fields became optional so earlier documents still parse. The committed schema was generated before that change. It still required the three figures and did not declare the flag, so it refused the diagnostics that UKFullGateKernel now writes. That kernel runs the UK build's full set of validation checks on the final calibrated weights and writes its calibration diagnostics. Regenerated with tools/generate_calibration_diagnostics_json_schema.py. Only the UKAreaFit definition changes: support_below_floor is an optional boolean or null, and each figure is optional, null or at least 0, as before. New tests check the published file the way consumers read it. It must accept a UK area fit with the floor flag only, with neither flag nor figures, with legacy figures, or with explicit nulls, and the models must accept each. It must also refuse a negative legacy figure, as the models do. Co-Authored-By: Claude Opus 5.5 --- .../calibration_diagnostics.v8.schema.json | 59 ++++++++++++++---- .../engine_free/shared/test_json_schema.py | 60 +++++++++++++++++++ 2 files changed, 106 insertions(+), 13 deletions(-) diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json index 6fb0660440..d33cc366f9 100644 --- a/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json @@ -751,20 +751,56 @@ "title": "Loss Contribution", "type": "number" }, + "support_below_floor": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Support Below Floor" + }, "nonzero_households": { - "minimum": 0, - "title": "Nonzero Households", - "type": "integer" + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Nonzero Households" }, "nonzero_source_households": { - "minimum": 0, - "title": "Nonzero Source Households", - "type": "integer" + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Nonzero Source Households" }, "effective_sample_size": { - "minimum": 0, - "title": "Effective Sample Size", - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Effective Sample Size" } }, "required": [ @@ -776,10 +812,7 @@ "pass_rate", "worst_target", "worst_abs_relative_error", - "loss_contribution", - "nonzero_households", - "nonzero_source_households", - "effective_sample_size" + "loss_contribution" ], "title": "UKAreaFit", "type": "object" diff --git a/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py index 5c58093267..2a437107d8 100644 --- a/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py +++ b/packages/microcosm-diagnostics/tests/engine_free/shared/test_json_schema.py @@ -21,6 +21,7 @@ from test_support.microcosm_diagnostics.json_schema import ( json_schema_errors, load_strict_json, + published_json_schema_errors, ) from test_support.microcosm_diagnostics.payloads import ( schema_8_payload, @@ -683,3 +684,62 @@ def test_generator_tool_writes_and_checks_the_committed_schema( assert "regenerate it with" in capsys.readouterr().err with pytest.raises(ValueError, match="differs from the schema the models generate"): diagnostics.assert_calibration_diagnostics_v8_json_schema_current(written) + + +LEGACY_AREA_SUPPORT_FIGURES = { + "nonzero_households": 60, + "nonzero_source_households": 55, + "effective_sample_size": 52.5, +} + + +def _uk_payload_with_area_fit(area_fields: dict[str, object]) -> dict: + """The complete UK document, its one weakest area carrying ``area_fields``.""" + + payload = uk_schema_8_payload() + (area,) = payload["uk_diagnostics"]["weakest_areas_by_fit"]["bottom_by_fit"] + del area["support_below_floor"] + area.update(area_fields) + return payload + + +@pytest.mark.parametrize( + "area_fields", + [ + pytest.param({"support_below_floor": True}, id="floor-flag-only"), + pytest.param({}, id="neither-flag-nor-figures"), + pytest.param(LEGACY_AREA_SUPPORT_FIGURES, id="legacy-support-figures"), + pytest.param( + {"support_below_floor": None} + | {field: None for field in LEGACY_AREA_SUPPORT_FIGURES}, + id="explicit-nulls", + ), + ], +) +def test_published_schema_accepts_uk_area_fits_with_the_floor_flag_or_legacy_support_figures( + area_fields: dict[str, object], +) -> None: + # Released diagnostics withhold an area's support figures and carry the + # floor flag instead (microcosm#1301); earlier documents carry the figures. + document = load_strict_json( + json.dumps(_uk_payload_with_area_fit(area_fields), allow_nan=False) + ) + diagnostics.CalibrationDiagnosticsV8.model_validate(document) + + assert published_json_schema_errors(document) == [] + + +def test_published_schema_and_models_refuse_a_negative_legacy_support_figure() -> None: + document = _uk_payload_with_area_fit( + LEGACY_AREA_SUPPORT_FIGURES | {"effective_sample_size": -1.0} + ) + + assert any( + error.startswith( + "$.uk_diagnostics.weakest_areas_by_fit.bottom_by_fit[0]" + ".effective_sample_size: " + ) + for error in published_json_schema_errors(document) + ) + with pytest.raises(ValidationError): + diagnostics.CalibrationDiagnosticsV8.model_validate(document) From 6df39673448d9e9e01a12c9efddf29c194da0593 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:13:59 +0400 Subject: [PATCH 4/4] Regenerate the schema-8 JSON Schema for withheld model figures and BRMA main's #1305 changed microcosm.diagnostics.schema: UK geography pass rates gain the brma level, and uk_diagnostics gains model_figures_withheld (UKWithheldModelFigures) for rows whose model figures a UK build withholds. The committed schema is regenerated with tools/generate_calibration_diagnostics_json_schema.py. Only three definitions change: UKWithheldModelFigures is new, UKDiagnosticsV1 gains model_figures_withheld, and UKGeographyPassRate accepts brma. The terminal kernel's schema test now also runs #1305's fixture with a withheld BRMA row, so a real producer covers both new parts. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/uk/test_uk_graph_terminal.py | 11 ++++- .../calibration_diagnostics.v8.schema.json | 47 ++++++++++++++++++- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py index 829db3893c..1a8a24d6ef 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py @@ -1115,13 +1115,20 @@ def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result( assert support_csv["support_below_floor"].dtype == bool +@pytest.mark.parametrize( + "brma_row", [False, True], ids=["published_rows", "withheld_brma_row"] +) def test_final_gate_kernel_diagnostics_match_the_published_json_schema( - monkeypatch, + monkeypatch, brma_row ): - stored = _run_final_gate_kernel(monkeypatch) + # The withheld BRMA row exercises the parts #1305 added: the ``brma`` + # geography level and ``uk_diagnostics.model_figures_withheld``. + stored = _run_final_gate_kernel(monkeypatch, brma_row=brma_row) document = load_strict_json(stored.artifacts["calibration_diagnostics"]) assert published_json_schema_errors(document) == [] + if brma_row: + assert document["uk_diagnostics"]["model_figures_withheld"]["targets"] == 1 def test_the_gate_kernel_keeps_a_withheld_brma_row_out_of_its_gates(monkeypatch): diff --git a/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json index d33cc366f9..fb3d185c53 100644 --- a/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json +++ b/packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json @@ -1006,6 +1006,17 @@ } ], "default": null + }, + "model_figures_withheld": { + "anyOf": [ + { + "$ref": "#/$defs/UKWithheldModelFigures" + }, + { + "type": "null" + } + ], + "default": null } }, "required": [ @@ -1064,7 +1075,8 @@ "region", "country", "local_authority", - "constituency" + "constituency", + "brma" ], "title": "Geography Level", "type": "string" @@ -1467,6 +1479,39 @@ "title": "UKWeightSummary", "type": "object" }, + "UKWithheldModelFigures": { + "additionalProperties": false, + "description": "Target rows whose model figures stay in the build (microcosm#1262).\n\nA row resting on fewer than ``min_source_households`` distinct source\nhouseholds keeps its place without its model figures. While any does, the\npayload's totals are restated over the published rows (``totals_basis``)\nso that none can be differenced against them, and its loss trajectory is\nempty.", + "properties": { + "targets": { + "exclusiveMinimum": 0, + "title": "Targets", + "type": "integer" + }, + "min_source_households": { + "exclusiveMinimum": 0, + "title": "Min Source Households", + "type": "integer" + }, + "totals_basis": { + "const": "published_rows", + "title": "Totals Basis", + "type": "string" + }, + "note": { + "title": "Note", + "type": "string" + } + }, + "required": [ + "targets", + "min_source_households", + "totals_basis", + "note" + ], + "title": "UKWithheldModelFigures", + "type": "object" + }, "UKZeroWeightStratum": { "additionalProperties": false, "properties": {