Publish a JSON Schema for calibration diagnostics schema 8

README.md

@@ -375,6 +375,13 @@ UK release workflows stop when diagnostics are unavailable because their later
 release checks require that evidence; historical schema-6 and schema-7 UK
 artifacts remain readable through isolated compatibility validation.
 
+A Draft 2020-12 JSON Schema for schema 8, generated from those models, is
+committed at
+`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json`
+for consumers that check the file without importing them, such as the
+collector. Its description lists the model rules a JSON Schema cannot express;
+see the [package README](packages/microcosm-diagnostics/README.md).
+
 Standard publication uploads the locally built `releases/<id>/` artifacts to
 the Hugging Face dataset, tags the release, and updates `latest.json`. It runs
 on the build machine (it needs the freshly built H5), so it isn't a CI step:

changelog.d/1293-calibration-diagnostics-json-schema-generator.added.md

@@ -0,0 +1,3 @@
+`microcosm.diagnostics.calibration_diagnostics_v8_json_schema()` generates a Draft 2020-12 JSON Schema for calibration diagnostics schema 8 from the pydantic models, so consumers such as the collector can check `calibration_diagnostics.json` without importing them (microcosm#1293). Its description lists every rule the models enforce that the schema cannot express: cross-field reconciliations, finite numbers, non-blank text, strict integers and digest anchoring under Python's `re`. It also notes the one case where the schema is stricter than the models: literal values. Tests pin that every model validator is listed, that each listed rule has a document the schema accepts and the models refuse, and that valid and representative invalid documents are classified the same way by both.
+
+`jsonschema`, already locked as a runtime dependency of `microcosm-build`, joins the `microcosm-diagnostics` dev group, so no new package enters the lock. The primary-QRF worker identity's approved `uv.lock` digest moves with the lock, as it does for every lock change.

changelog.d/1293-calibration-diagnostics-json-schema.added.md

@@ -0,0 +1 @@
+The calibration-diagnostics schema-8 JSON Schema is committed at `packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json` and ships in the `microcosm-diagnostics` wheel (microcosm#1293). `tools/generate_calibration_diagnostics_json_schema.py` regenerates it, and `--check` reports a stale file. A byte-exact drift test keeps the file equal to freshly generated output. In their engine-free tests, the US national release writer, the ACS local-area calibrate stage, the UK writer, the UK graph terminal gate kernel and the transport terminal kernel now validate the diagnostics they write against the committed file.

docs/agent-guide.md

@@ -413,6 +413,17 @@ label, search for an existing definition and follow
 AI assistants must import shared static data from its domain-specific constants
 module instead of copying it or reconstructing alternate views in consumers.
 
+## Calibration diagnostics JSON Schema
+
+`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json`
+is generated from the schema-8 models in `microcosm.diagnostics.schema`. After
+changing one, regenerate it with
+`uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`.
+A new model validator also needs an entry in
+`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS` and a document in
+`test_json_schema.py` that the schema accepts and the models refuse; the
+diagnostics tests fail until both exist.
+
 ## Review this guide
 
 Update this guide in the same PR whenever the workspace layout, test

packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py

@@ -28,7 +28,7 @@
 PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker"
 PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}"
 APPROVED_UV_LOCK_SHA256 = (
-    "e5d5818bdc73fd44fbb474b6f0bca4b697ee2f70a0cbb9e47eb04b20f7408d77"
+    "751721b231ab8cc6b3d10164f61ca429f5171661951494503fb0873e3f67b117"
 )
 LEGACY_CAMPAIGN_UV_LOCK_SHA256 = (
     "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee"

packages/microcosm-build/tests/engine_free/shared/test_transport_terminal_kernels.py

@@ -88,6 +88,10 @@
     with_config,
     write_toy_sources,
 )
+from test_support.microcosm_diagnostics.json_schema import (
+    load_strict_json,
+    published_json_schema_errors,
+)
 
 DIAGNOSTICS = "toy.calibration.diagnostics"
 _KEY = "b" * 64
@@ -179,6 +183,12 @@ def test_diagnostics_validate_against_schema_8(toy_run, tmp_path) -> None:
     )
 
 
+def test_transport_diagnostics_match_the_published_json_schema(toy_run) -> None:
+    payload = toy_run.artifact(DIAGNOSTICS, "diagnostics")
+
+    assert published_json_schema_errors(load_strict_json(payload)) == []
+
+
 def _diagnostics_artifacts(run, **overrides) -> dict[str, ArtifactValue]:
     artifacts = {
         alias: _value(run.artifact(producer, name), kind)

packages/microcosm-build/tests/engine_free/uk/test_uk_diagnostics.py

@@ -37,6 +37,10 @@
     DiagnosticsWriteSuccess,
 )
 from microcosm.frame import EntitySchema, Frame, WeightKind, Weights
+from test_support.microcosm_diagnostics.json_schema import (
+    load_strict_json,
+    published_json_schema_errors,
+)
 
 _SPI_COLUMN = "household_is_spi_synthetic"
 _CG_COLUMN = "household_is_capital_gains_clone"
@@ -957,6 +961,37 @@ def test_writer_round_trips_strict_json(tmp_path: Path) -> None:
     assert list(tmp_path.glob(f".{path.name}.*.tmp")) == []
 
 
+@pytest.mark.parametrize(
+    ("with_skipped", "with_consumption"),
+    [
+        pytest.param(False, False, id="scored-targets-only"),
+        pytest.param(True, True, id="skipped-target-and-consumption-drift"),
+    ],
+)
+def test_written_uk_diagnostics_match_the_published_json_schema(
+    tmp_path: Path,
+    with_skipped: bool,
+    with_consumption: bool,
+) -> None:
+    result, frame, registry, geography = _diagnostics_case(
+        with_skipped=with_skipped,
+        with_consumption=with_consumption,
+    )
+    path = tmp_path / "calibration_diagnostics.json"
+
+    outcome = write_uk_calibration_diagnostics(
+        result,
+        path,
+        frame,
+        target_geography_levels=geography,
+        target_registry=registry,
+        build={"release_id": "fixture"},
+    )
+
+    assert isinstance(outcome, DiagnosticsWriteSuccess)
+    assert published_json_schema_errors(load_strict_json(path.read_bytes())) == []
+
+
 @pytest.mark.parametrize(
     "value",
     [float("nan"), float("inf"), float("-inf")],

packages/microcosm-build/tests/engine_free/uk/test_uk_graph_terminal.py

@@ -21,6 +21,10 @@
 from test_support.microcosm_build.uk_hierarchy_fixtures import (
     uk_fixture_hierarchy,
 )
+from test_support.microcosm_diagnostics.json_schema import (
+    load_strict_json,
+    published_json_schema_errors,
+)
 
 
 def test_export_drops_native_aliases_and_keeps_identity_keyed_assignment(tmp_path):
@@ -950,6 +954,9 @@ def test_both_gate_kernels_are_registered():
 def _run_final_gate_kernel(monkeypatch, *, brma_row: bool = False, solved=None):
     """Run the terminal gate kernel on a two-household fixture.
 
+    Rebuilding the diagnostics is forbidden, so a returned document proves
+    the kernel reused the decoded calibration result.
+
     ``solved``, a list, receives the calibration result the kernel decodes.
 
     ``brma_row`` adds a Scottish BRMA rent row far from its target: its two
@@ -1267,6 +1274,22 @@ def test_final_gate_kernel_owns_complete_diagnostics_and_reuses_decoded_result(
     assert support_csv["support_below_floor"].dtype == bool
 
 
+@pytest.mark.parametrize(
+    "brma_row", [False, True], ids=["published_rows", "withheld_brma_row"]
+)
+def test_final_gate_kernel_diagnostics_match_the_published_json_schema(
+    monkeypatch, brma_row
+):
+    # The withheld BRMA row exercises the parts #1305 added: the ``brma``
+    # geography level and ``uk_diagnostics.model_figures_withheld``.
+    stored = _run_final_gate_kernel(monkeypatch, brma_row=brma_row)
+    document = load_strict_json(stored.artifacts["calibration_diagnostics"])
+
+    assert published_json_schema_errors(document) == []
+    if brma_row:
+        assert document["uk_diagnostics"]["model_figures_withheld"]["targets"] == 1
+
+
 def test_the_gate_kernel_keeps_a_withheld_brma_row_out_of_its_gates(monkeypatch):
     """The third review of #1305: the real bindings, through the kernel.
 

packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py

@@ -36,6 +36,10 @@
 from test_support.microcosm_build.us_batched_target_materialization import (
     _nested_frame,
 )
+from test_support.microcosm_diagnostics.json_schema import (
+    load_strict_json,
+    published_json_schema_errors,
+)
 from test_support.paths import paths_for
 
 _TEST_PATHS = paths_for("microcosm-build")
@@ -401,6 +405,13 @@ def test_every_calibrate_output_records_the_weights(calibrate_run) -> None:
     )
 
 
+def test_calibrate_diagnostics_match_the_published_json_schema(calibrate_run) -> None:
+    calibrate_run.run()
+    written = calibrate_run.path / "calibration_diagnostics.json"
+
+    assert published_json_schema_errors(load_strict_json(written.read_bytes())) == []
+
+
 def test_a_family_multiplier_reaches_calibrate_and_the_stamp(calibrate_run) -> None:
     calls = calibrate_run.run(target_family_loss_multipliers={"usda_snap": 4.0})
     train = _train_specs(calibrate_run)

packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py

@@ -27,6 +27,10 @@
 )
 from microcosm.data.stored_inputs import register_sha256
 from microcosm.frame import EntitySchema, Frame, WeightKind, Weights
+from test_support.microcosm_diagnostics.json_schema import (
+    load_strict_json,
+    published_json_schema_errors,
+)
 from test_support.paths import paths_for
 
 _TEST_PATHS = paths_for("microcosm-build")
@@ -5895,10 +5899,9 @@ def fake_write_calibration_diagnostics(
     }
 
 
-def test_release_calibration_diagnostics_writes_nan_final_loss_as_null(
-    small_frame,
-    tmp_path,
-) -> None:
+def _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path) -> Path:
+    """Write release diagnostics for a one-target US registry with a NaN loss."""
+
     builder = _load_builder_module()
     base_h5 = tmp_path / "base.h5"
     base_h5.write_bytes(b"h5")
@@ -5971,8 +5974,16 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null(
         # writer; keep that call boundary explicit rather than moving the fix.
         default_dataset={"method": "dense_no_l0", "final_loss": None},
     )
+    return tmp_path / "calibration_diagnostics.json"
+
+
+def test_release_calibration_diagnostics_writes_nan_final_loss_as_null(
+    small_frame,
+    tmp_path,
+) -> None:
+    written = _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path)
 
-    diagnostics = json.loads((tmp_path / "calibration_diagnostics.json").read_text())
+    diagnostics = json.loads(written.read_text())
     assert diagnostics["schema_version"] == 8
     assert diagnostics["targets"][0]["source"] == "fixture"
     assert diagnostics["targets"][0]["hierarchy"] == {
@@ -5997,6 +6008,15 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null(
     assert diagnostics["build"]["default_dataset"]["final_loss"] is None
 
 
+def test_release_calibration_diagnostics_match_the_published_json_schema(
+    small_frame,
+    tmp_path,
+) -> None:
+    written = _write_release_diagnostics_with_nan_final_loss(small_frame, tmp_path)
+
+    assert published_json_schema_errors(load_strict_json(written.read_bytes())) == []
+
+
 #: What the stubbed stored-input gate refuses in the ``stored_input_refused``
 #: harness mode, in the gate's own line format.
 STORED_INPUT_REFUSAL_FIXTURE = (

packages/microcosm-diagnostics/README.md

@@ -26,3 +26,24 @@ reach validation as non-finite numbers fail construction. The writer also uses
 strict JSON serialization with non-finite output disabled, so a value introduced
 after model construction returns a structured serialization failure instead of
 silently becoming `null`.
+
+`calibration_diagnostics_v8_json_schema()` generates a Draft 2020-12 JSON
+Schema for schema 8 from `CalibrationDiagnosticsV8`, for consumers that check
+the file without importing these models. The models remain the authority.
+Their cross-field reconciliations, finite-number and non-blank-text checks,
+and strict integers cannot be written in a schema generated this way. The
+schema's description lists each of these rules
+(`CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS`), and a test fails if a model
+validator is added without being listed. Complete validation still goes
+through `parse_calibration_diagnostics`.
+
+The generated schema is committed, and shipped in the wheel, at
+
+`packages/microcosm-diagnostics/src/microcosm/diagnostics/schemas/calibration_diagnostics.v8.schema.json`
+
+`load_calibration_diagnostics_v8_json_schema()` reads it. After changing a
+schema-8 model, regenerate it with
+`uv run --no-sync python tools/generate_calibration_diagnostics_json_schema.py`;
+`--check` reports a stale file without writing. A test fails while the
+committed file differs from freshly generated output, and the US, UK and
+transport build tests validate the diagnostics they write against it.

packages/microcosm-diagnostics/pyproject.toml

@@ -13,7 +13,9 @@ Homepage = "https://microcosm.institute"
 Repository = "https://github.com/PolicyEngine/microcosm"
 
 [dependency-groups]
-dev = ["pytest>=8"]
+# jsonschema checks documents against the generated schema-8 JSON Schema in
+# tests only; the package itself needs nothing beyond pydantic.
+dev = ["pytest>=8", "jsonschema>=4.23,<5"]
 
 [build-system]
 requires = ["hatchling"]

packages/microcosm-diagnostics/src/microcosm/diagnostics/__init__.py

@@ -1,5 +1,19 @@
 """Typed calibration diagnostics shared across the Microcosm stack."""
 
+from microcosm.diagnostics.json_schema import (
+    CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS,
+    CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID,
+    CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE,
+    CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE,
+    JSON_SCHEMA_DRAFT_2020_12,
+    JsonSchemaGap,
+    assert_calibration_diagnostics_v8_json_schema_current,
+    calibration_diagnostics_v8_json_schema,
+    committed_calibration_diagnostics_v8_json_schema_path,
+    load_calibration_diagnostics_v8_json_schema,
+    render_calibration_diagnostics_v8_json_schema,
+    write_calibration_diagnostics_v8_json_schema,
+)
 from microcosm.diagnostics.schema import (
     CALIBRATION_DIAGNOSTICS_ADAPTER,
     CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION,
@@ -41,13 +55,19 @@
 __all__ = [
     "CALIBRATION_DIAGNOSTICS_ADAPTER",
     "CALIBRATION_DIAGNOSTICS_SCHEMA_VERSION",
+    "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_GAPS",
+    "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_ID",
+    "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_LITERAL_NOTE",
+    "CALIBRATION_DIAGNOSTICS_V8_JSON_SCHEMA_RESOURCE",
+    "JSON_SCHEMA_DRAFT_2020_12",
     "SUPPORTED_CALIBRATION_DIAGNOSTICS_SCHEMA_VERSIONS",
     "CalibrationDiagnostics",
     "CalibrationDiagnosticsV8",
     "DiagnosticsFailureCode",
     "DiagnosticsWriteFailure",
     "DiagnosticsWriteOutcome",
     "DiagnosticsWriteSuccess",
+    "JsonSchemaGap",
     "UK_DIAGNOSTICS_SCHEMA_VERSION",
     "UK_MODEL_FIGURE_FIELDS",
     "UK_OPTIONAL_TARGET_GEOGRAPHY_LEVELS",
@@ -68,10 +88,16 @@
     "UKWeightSummary",
     "UKWithheldModelFigures",
     "UKZeroWeightStratum",
+    "assert_calibration_diagnostics_v8_json_schema_current",
     "calibration_diagnostics_json_schema",
+    "calibration_diagnostics_v8_json_schema",
+    "committed_calibration_diagnostics_v8_json_schema_path",
     "failed_diagnostics_outcome",
+    "load_calibration_diagnostics_v8_json_schema",
     "parse_calibration_diagnostics",
+    "render_calibration_diagnostics_v8_json_schema",
     "write_calibration_diagnostics",
+    "write_calibration_diagnostics_v8_json_schema",
 ]
 
 __version__ = "0.1.0"