Bound native evidence by bytes, not by the schema item budget

changelog.d/1144-native-evidence-bounds.fixed.md

@@ -0,0 +1 @@
+Bound native evidence files by bytes and nesting depth instead of the compiler-schema item budget, and hash checkpoint and published evidence files without parsing them, so a licensed spine manifest (over a million JSON items in 13 MB) no longer refuses the dense build's checkpoint links or the export of its own evidence.

docs/orrery-adapter.md

@@ -207,7 +207,7 @@ build into the same output directory therefore keeps the record of the attempt
 that computed a result next to the replay that reused it; a published file
 that no longer verifies refuses publication instead of being dropped. `microcosm.graph.run-binding.v1` records the graph/manifest byte
 digests, manifest key, attempt, phase, platform and identities of used sources.
-These native files support later export; they do not create an Orrery snapshot.
+These native files support later export; they do not create an Orrery snapshot. Native evidence files are bounded by bytes (1 GiB) and nesting depth; the compiler schema's per-item budget applies to schemas and Orrery documents, not to copied manifests, which hold over a million JSON items on a licensed spine. The byte cap is set by the largest file a real build writes — the K=25/60k dense build's numerical and preflight graphs are 399 MiB (41.4M items, 2.2 GiB peak RSS to parse) — with about 2.5x headroom. Digest checks read bytes without parsing, so hashing a graph that size never pays the parse cost.
 A serialized historical manifest without this binding cannot supply missing
 run-end source identities by reading today's source files.
 

packages/microcosm-graph/src/microcosm/graph/evidence.py

@@ -10,6 +10,7 @@
 import hashlib
 import json
 import logging
+import math
 import os
 import stat
 import uuid
@@ -52,7 +53,18 @@
 BINDING_PROTOCOL = "microcosm.graph.run-binding.v1"
 EVIDENCE_INPUT_PROTOCOL = "microcosm.graph.execution-input.v1"
 EXECUTION_PROTOCOL = "microcosm.graph.execution-evidence.v1"
-_MAX_BYTES = 32 * 1024 * 1024
+# Native evidence files are exact copies of run manifests and graphs, which a
+# real build already holds in memory; they are bounded by bytes and nesting
+# depth only. The per-item complexity charge of the compiler-schema reader is
+# not applied here: a licensed spine manifest has over a million JSON items in
+# 13 MB and must remain loadable and hashable. The cap must admit every file a
+# real build writes: the K=25/60k dense build's numerical/preflight graphs are
+# 399 MiB (41.4M items, measured 2.2 GiB peak RSS to parse — about 5.7x the
+# bytes, not the 25-35x of a tiny-item shape, which no real evidence file
+# has). 1 GiB is that largest real file with ~2.5x headroom; digest-only
+# paths use :func:`read_bytes` and never pay the parse cost.
+_MAX_BYTES = 1024 * 1024 * 1024
+_MAX_DEPTH = 64
 _SUMMARY_FIELDS = frozenset({"node", "artifact", "key", "type", "data"})
 _LOG = logging.getLogger(__name__)
 
@@ -73,17 +85,52 @@ def _unique(pairs):
     return result
 
 
-def read_json(path: Path) -> tuple[object, bytes]:
-    """Bounded regular-file reader used for all native evidence inputs."""
+def _reject_constant(_value: str) -> None:
+    raise ValueError("Graph evidence: evidence input requires finite JSON numbers.")
+
+
+def _bounded_depth(value: object) -> None:
+    pending = [(value, 0)]
+    while pending:
+        item, depth = pending.pop()
+        require(depth <= _MAX_DEPTH, "evidence input nesting limit")
+        if type(item) is list:
+            pending.extend((child, depth + 1) for child in item)
+        elif type(item) is dict:
+            pending.extend((child, depth + 1) for child in item.values())
+        elif type(item) is float:
+            # ``parse_constant`` only fires on the NaN/Infinity literals; an
+            # overflowing number literal such as 1e400 reaches here as inf.
+            require(
+                math.isfinite(item),
+                "Graph evidence: evidence input requires finite JSON numbers.",
+            )
+
+
+def read_bytes(path: Path) -> bytes:
+    """Bounded regular-file reader for hashing or copying native evidence."""
     descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
     with os.fdopen(descriptor, "rb") as stream:
         info = os.fstat(stream.fileno())
         require(stat.S_ISREG(info.st_mode), "evidence input must be a regular file")
         require(info.st_size <= _MAX_BYTES, "evidence input byte limit")
         payload = stream.read(_MAX_BYTES + 1)
     require(len(payload) <= _MAX_BYTES, "evidence input byte limit")
-    value = json.loads(payload, object_pairs_hook=_unique)
-    return _plain_json(value), payload
+    return payload
+
+
+def read_json(path: Path) -> tuple[object, bytes]:
+    """Bounded regular-file reader used for all native evidence inputs.
+
+    Duplicate keys, non-finite numbers and nesting beyond the depth limit are
+    refused; item count is not charged, only bytes.
+    """
+    payload = read_bytes(path)
+    value = json.loads(
+        payload, object_pairs_hook=_unique, parse_constant=_reject_constant
+    )
+    _bounded_depth(value)
+    return value, payload
 
 
 @dataclass(frozen=True)
@@ -752,7 +799,7 @@ def checkpoint_references(
             missing.append(f"{label} bytes unavailable (sha256 {recorded})")
             continue
         require(
-            sha256(read_json(path)[1]) == recorded,
+            sha256(read_bytes(path)) == recorded,
             f"recorded checkpoint {label.lower()} digest mismatch",
         )
         refs.append({"label": label, "url": str(path.resolve()), "sha256": recorded})
@@ -790,7 +837,7 @@ def save_graph_schema(
                 source = Path(url)
                 if not source.is_absolute() or not source.is_file():
                     continue
-                payload = read_json(source)[1]
+                payload = read_bytes(source)
                 require(
                     sha256(payload) == digest(ref.get("sha256")),
                     "checkpoint reference changed during capture",
@@ -864,7 +911,7 @@ def publish_run_evidence(
                     "restore it or clear the published evidence",
                 )
                 require(
-                    sha256(read_json(target)[1]) == digest(ref["sha256"]),
+                    sha256(read_bytes(target)) == digest(ref["sha256"]),
                     f"published evidence file {relative} no longer matches its digest",
                 )
             runs.append(entry)
@@ -880,7 +927,7 @@ def publish_run_evidence(
             filename = (
                 f"{prefix}-{entry['attempt_id']}-{relative.parent.name}-{field}.json"
             )
-            payload = read_json(index_path.parent / relative)[1]
+            payload = read_bytes(index_path.parent / relative)
             require(
                 sha256(payload) == digest(entry[field]["sha256"]),
                 "native phase evidence changed during capture",

packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py

@@ -636,3 +636,83 @@ def test_malformed_composite_metadata_is_refused(fresh, composite):
     contract["operations"]["survey"]["composite"] = composite
     with pytest.raises(ValueError, match="composite"):
         orrery_document(fresh.compiled, extensions={PRESENTATION_EXTENSION: contract})
+
+
+def test_item_heavy_native_evidence_is_read_hashed_and_copied(fresh, tmp_path):
+    """A real spine manifest holds over a million small JSON items in 13 MB."""
+    from microcosm.graph.evidence import read_bytes, read_json
+    from microcosm.graph.presentation import PRESENTATION_EXTENSION
+
+    heavy = tmp_path / "manifest.json"
+    heavy.write_bytes(json.dumps({"receipts": [[0, 1]] * 1_500_000}).encode())
+    assert heavy.stat().st_size < 32 * 1024 * 1024
+    value, payload = read_json(heavy)
+    assert len(value["receipts"]) == 1_500_000
+    assert sha256(payload) == sha256(read_bytes(heavy))
+    record = {
+        "graph_declaration": {"path": str(heavy), "sha256": sha256(payload)},
+        "graph_manifest": {"path": str(heavy), "sha256": sha256(payload)},
+    }
+    refs, missing = checkpoint_references(record, base=tmp_path)
+    assert len(refs) == 2 and missing is None
+    contract = presentation(fresh.compiled)
+    contract["scope"]["boundaries"] = [
+        {"operation": "survey", "kind": "checkpoint", "upstream": refs}
+    ]
+    saved = save_graph_schema(fresh.compiled, tmp_path / "out", presentation=contract)
+    copied = json.loads(saved.read_bytes())["extensions"][PRESENTATION_EXTENSION]
+    url = copied["scope"]["boundaries"][0]["upstream"][0]["url"]
+    assert (saved.parent / url).read_bytes() == payload
+
+
+def test_item_heavy_manifest_round_trips_through_run_evidence(fresh, tmp_path):
+    """The second #1187 failure: save, load and publish an item-heavy manifest."""
+    from dataclasses import replace
+
+    name, receipt = next(iter(fresh.manifest.nodes.items()))
+    heavy_manifest = replace(
+        fresh.manifest,
+        nodes={
+            **fresh.manifest.nodes,
+            name: replace(receipt, receipt={"receipts": [[0, 1]] * 1_500_000}),
+        },
+    )
+    index = save_run_evidence(
+        fresh.compiled,
+        heavy_manifest,
+        store=fresh.store,
+        directory=tmp_path / "attempt-1",
+        attempt_id="attempt-1",
+        phase="numerical",
+    )
+    written = next((tmp_path / "attempt-1").rglob("manifest.json"))
+    assert written.stat().st_size > 10 * 1024 * 1024
+    runs = load_run_evidence(index, store=fresh.store)
+    assert len(runs[0].manifest.nodes[name].receipt["receipts"]) == 1_500_000
+    published = publish_run_evidence(index, tmp_path / "out")
+    entry = json.loads(published.read_bytes())["runs"][0]
+    copied = published.parent / entry["manifest"]["path"]
+    assert copied.read_bytes() == written.read_bytes()
+    republished = load_run_evidence(published, store=fresh.store)
+    assert (
+        republished[0].manifest.nodes[name].receipt["receipts"][0]
+        == runs[0].manifest.nodes[name].receipt["receipts"][0]
+    )
+
+
+@pytest.mark.parametrize(
+    "text, message",
+    [
+        ("[NaN]", "finite"),
+        ("[1e400]", "finite"),
+        ("[" * 70 + "]" * 70, "nesting"),
+        ('{"a": 1, "a": 2}', "duplicate"),
+    ],
+)
+def test_native_evidence_reader_refuses_malformed_json(tmp_path, text, message):
+    from microcosm.graph.evidence import read_json
+
+    path = tmp_path / "bad.json"
+    path.write_text(text)
+    with pytest.raises(ValueError, match=message):
+        read_json(path)