From 9d423582ef22753367e65c68ce622e58fd02a7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:04:26 +0100 Subject: [PATCH 1/2] Bound native evidence by bytes, not by the schema item budget The f10 rung on the merged #1144 drivers refused before its first phase: "Graph schema exceeds the metadata size limit". The licensed spine's run manifest is 12.5 MB but holds 1.42 million JSON items, and the native evidence reader charged 16 bytes per item, counting it as 48.9 MiB against a 32 MiB budget. The dense driver only wanted to hash that file for its checkpoint link, and the same reader guards load_run_evidence, so the spine's own evidence could not be exported either. Native evidence files are now bounded by bytes (1 GiB) and nesting depth, with duplicate keys and non-finite numbers still refused; the per-item budget stays with compiler schemas and Orrery documents. Checkpoint links, copied upstream bytes and published evidence files are hashed without parsing. On the real spine bundle the export path now loads in 11 s and projects in 1.5 s. Co-Authored-By: Claude Fable 5.1 --- .../1144-native-evidence-bounds.fixed.md | 1 + docs/orrery-adapter.md | 2 +- .../src/microcosm/graph/evidence.py | 52 +++++++++++++++---- .../engine_free/shared/test_graph_evidence.py | 44 ++++++++++++++++ 4 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 changelog.d/1144-native-evidence-bounds.fixed.md diff --git a/changelog.d/1144-native-evidence-bounds.fixed.md b/changelog.d/1144-native-evidence-bounds.fixed.md new file mode 100644 index 0000000000..cc0292e4ba --- /dev/null +++ b/changelog.d/1144-native-evidence-bounds.fixed.md @@ -0,0 +1 @@ +Bound native evidence files by bytes and nesting depth instead of the compiler-schema item budget, and hash checkpoint and published evidence files without parsing them, so a licensed spine manifest (over a million JSON items in 13 MB) no longer refuses the dense build's checkpoint links or the export of its own evidence. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index 7f1f8f7ae5..150479f0bd 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -207,7 +207,7 @@ build into the same output directory therefore keeps the record of the attempt that computed a result next to the replay that reused it; a published file that no longer verifies refuses publication instead of being dropped. `microcosm.graph.run-binding.v1` records the graph/manifest byte digests, manifest key, attempt, phase, platform and identities of used sources. -These native files support later export; they do not create an Orrery snapshot. +These native files support later export; they do not create an Orrery snapshot. Native evidence files are bounded by bytes (1 GiB) and nesting depth; the compiler schema's per-item budget applies to schemas and Orrery documents, not to copied manifests, which hold over a million JSON items on a licensed spine. Digest checks read bytes without parsing. A serialized historical manifest without this binding cannot supply missing run-end source identities by reading today's source files. diff --git a/packages/microcosm-graph/src/microcosm/graph/evidence.py b/packages/microcosm-graph/src/microcosm/graph/evidence.py index bbba7501c1..e470f24dcf 100644 --- a/packages/microcosm-graph/src/microcosm/graph/evidence.py +++ b/packages/microcosm-graph/src/microcosm/graph/evidence.py @@ -52,7 +52,13 @@ BINDING_PROTOCOL = "microcosm.graph.run-binding.v1" EVIDENCE_INPUT_PROTOCOL = "microcosm.graph.execution-input.v1" EXECUTION_PROTOCOL = "microcosm.graph.execution-evidence.v1" -_MAX_BYTES = 32 * 1024 * 1024 +# Native evidence files are exact copies of run manifests and graphs, which a +# real build already holds in memory; they are bounded by bytes and nesting +# depth only. The per-item complexity charge of the compiler-schema reader is +# not applied here: a licensed spine manifest has over a million JSON items in +# 13 MB and must remain loadable and hashable. +_MAX_BYTES = 1024 * 1024 * 1024 +_MAX_DEPTH = 64 _SUMMARY_FIELDS = frozenset({"node", "artifact", "key", "type", "data"}) _LOG = logging.getLogger(__name__) @@ -73,8 +79,23 @@ def _unique(pairs): return result -def read_json(path: Path) -> tuple[object, bytes]: - """Bounded regular-file reader used for all native evidence inputs.""" +def _reject_constant(_value: str) -> None: + raise ValueError("Graph evidence: evidence input requires finite JSON numbers.") + + +def _bounded_depth(value: object) -> None: + pending = [(value, 0)] + while pending: + item, depth = pending.pop() + require(depth <= _MAX_DEPTH, "evidence input nesting limit") + if type(item) is list: + pending.extend((child, depth + 1) for child in item) + elif type(item) is dict: + pending.extend((child, depth + 1) for child in item.values()) + + +def read_bytes(path: Path) -> bytes: + """Bounded regular-file reader for hashing or copying native evidence.""" descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) with os.fdopen(descriptor, "rb") as stream: info = os.fstat(stream.fileno()) @@ -82,8 +103,21 @@ def read_json(path: Path) -> tuple[object, bytes]: require(info.st_size <= _MAX_BYTES, "evidence input byte limit") payload = stream.read(_MAX_BYTES + 1) require(len(payload) <= _MAX_BYTES, "evidence input byte limit") - value = json.loads(payload, object_pairs_hook=_unique) - return _plain_json(value), payload + return payload + + +def read_json(path: Path) -> tuple[object, bytes]: + """Bounded regular-file reader used for all native evidence inputs. + + Duplicate keys, non-finite numbers and nesting beyond the depth limit are + refused; item count is not charged, only bytes. + """ + payload = read_bytes(path) + value = json.loads( + payload, object_pairs_hook=_unique, parse_constant=_reject_constant + ) + _bounded_depth(value) + return value, payload @dataclass(frozen=True) @@ -752,7 +786,7 @@ def checkpoint_references( missing.append(f"{label} bytes unavailable (sha256 {recorded})") continue require( - sha256(read_json(path)[1]) == recorded, + sha256(read_bytes(path)) == recorded, f"recorded checkpoint {label.lower()} digest mismatch", ) refs.append({"label": label, "url": str(path.resolve()), "sha256": recorded}) @@ -790,7 +824,7 @@ def save_graph_schema( source = Path(url) if not source.is_absolute() or not source.is_file(): continue - payload = read_json(source)[1] + payload = read_bytes(source) require( sha256(payload) == digest(ref.get("sha256")), "checkpoint reference changed during capture", @@ -864,7 +898,7 @@ def publish_run_evidence( "restore it or clear the published evidence", ) require( - sha256(read_json(target)[1]) == digest(ref["sha256"]), + sha256(read_bytes(target)) == digest(ref["sha256"]), f"published evidence file {relative} no longer matches its digest", ) runs.append(entry) @@ -880,7 +914,7 @@ def publish_run_evidence( filename = ( f"{prefix}-{entry['attempt_id']}-{relative.parent.name}-{field}.json" ) - payload = read_json(index_path.parent / relative)[1] + payload = read_bytes(index_path.parent / relative) require( sha256(payload) == digest(entry[field]["sha256"]), "native phase evidence changed during capture", diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py index 356a760f35..1baba63d3f 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py @@ -636,3 +636,47 @@ def test_malformed_composite_metadata_is_refused(fresh, composite): contract["operations"]["survey"]["composite"] = composite with pytest.raises(ValueError, match="composite"): orrery_document(fresh.compiled, extensions={PRESENTATION_EXTENSION: contract}) + + +def test_item_heavy_native_evidence_is_read_hashed_and_copied(fresh, tmp_path): + """A real spine manifest holds over a million small JSON items in 13 MB.""" + from microcosm.graph.evidence import read_bytes, read_json + from microcosm.graph.presentation import PRESENTATION_EXTENSION + + heavy = tmp_path / "manifest.json" + heavy.write_bytes(json.dumps({"receipts": [[0, 1]] * 1_500_000}).encode()) + assert heavy.stat().st_size < 32 * 1024 * 1024 + value, payload = read_json(heavy) + assert len(value["receipts"]) == 1_500_000 + assert sha256(payload) == sha256(read_bytes(heavy)) + record = { + "graph_declaration": {"path": str(heavy), "sha256": sha256(payload)}, + "graph_manifest": {"path": str(heavy), "sha256": sha256(payload)}, + } + refs, missing = checkpoint_references(record, base=tmp_path) + assert len(refs) == 2 and missing is None + contract = presentation(fresh.compiled) + contract["scope"]["boundaries"] = [ + {"operation": "survey", "kind": "checkpoint", "upstream": refs} + ] + saved = save_graph_schema(fresh.compiled, tmp_path / "out", presentation=contract) + copied = json.loads(saved.read_bytes())["extensions"][PRESENTATION_EXTENSION] + url = copied["scope"]["boundaries"][0]["upstream"][0]["url"] + assert (saved.parent / url).read_bytes() == payload + + +@pytest.mark.parametrize( + "text, message", + [ + ("[NaN]", "finite"), + ("[" * 70 + "]" * 70, "nesting"), + ('{"a": 1, "a": 2}', "duplicate"), + ], +) +def test_native_evidence_reader_refuses_malformed_json(tmp_path, text, message): + from microcosm.graph.evidence import read_json + + path = tmp_path / "bad.json" + path.write_text(text) + with pytest.raises(ValueError, match=message): + read_json(path) From 3bcc3d002841905e489b9e486e15529d370fa2e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:54:42 +0100 Subject: [PATCH 2/2] Refuse overflowing number literals and pin the publish round trip (#1188 review) parse_constant only fires on the NaN/Infinity literals, so 1e400 reached the evidence value as inf; _bounded_depth now refuses any non-finite float it visits, at no extra cost on finite input. The byte cap stays at 1 GiB with its basis measured and recorded: the K=25/60k dense build's numerical and preflight graphs are 399 MiB (41.4M items, 2.2 GiB peak RSS to parse, ~5.7x the bytes), so 32 or 128 MiB would refuse real builds the way #1187 did, while digest paths hash bytes without parsing. A new round-trip test saves, loads and publishes an item-heavy manifest shaped like the licensed spine's and fails on the pre-#1188 reader. Co-Authored-By: Claude Fable 5 --- docs/orrery-adapter.md | 2 +- .../src/microcosm/graph/evidence.py | 15 +++++++- .../engine_free/shared/test_graph_evidence.py | 36 +++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index 150479f0bd..b095093260 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -207,7 +207,7 @@ build into the same output directory therefore keeps the record of the attempt that computed a result next to the replay that reused it; a published file that no longer verifies refuses publication instead of being dropped. `microcosm.graph.run-binding.v1` records the graph/manifest byte digests, manifest key, attempt, phase, platform and identities of used sources. -These native files support later export; they do not create an Orrery snapshot. Native evidence files are bounded by bytes (1 GiB) and nesting depth; the compiler schema's per-item budget applies to schemas and Orrery documents, not to copied manifests, which hold over a million JSON items on a licensed spine. Digest checks read bytes without parsing. +These native files support later export; they do not create an Orrery snapshot. Native evidence files are bounded by bytes (1 GiB) and nesting depth; the compiler schema's per-item budget applies to schemas and Orrery documents, not to copied manifests, which hold over a million JSON items on a licensed spine. The byte cap is set by the largest file a real build writes — the K=25/60k dense build's numerical and preflight graphs are 399 MiB (41.4M items, 2.2 GiB peak RSS to parse) — with about 2.5x headroom. Digest checks read bytes without parsing, so hashing a graph that size never pays the parse cost. A serialized historical manifest without this binding cannot supply missing run-end source identities by reading today's source files. diff --git a/packages/microcosm-graph/src/microcosm/graph/evidence.py b/packages/microcosm-graph/src/microcosm/graph/evidence.py index e470f24dcf..8b887a05b7 100644 --- a/packages/microcosm-graph/src/microcosm/graph/evidence.py +++ b/packages/microcosm-graph/src/microcosm/graph/evidence.py @@ -10,6 +10,7 @@ import hashlib import json import logging +import math import os import stat import uuid @@ -56,7 +57,12 @@ # real build already holds in memory; they are bounded by bytes and nesting # depth only. The per-item complexity charge of the compiler-schema reader is # not applied here: a licensed spine manifest has over a million JSON items in -# 13 MB and must remain loadable and hashable. +# 13 MB and must remain loadable and hashable. The cap must admit every file a +# real build writes: the K=25/60k dense build's numerical/preflight graphs are +# 399 MiB (41.4M items, measured 2.2 GiB peak RSS to parse — about 5.7x the +# bytes, not the 25-35x of a tiny-item shape, which no real evidence file +# has). 1 GiB is that largest real file with ~2.5x headroom; digest-only +# paths use :func:`read_bytes` and never pay the parse cost. _MAX_BYTES = 1024 * 1024 * 1024 _MAX_DEPTH = 64 _SUMMARY_FIELDS = frozenset({"node", "artifact", "key", "type", "data"}) @@ -92,6 +98,13 @@ def _bounded_depth(value: object) -> None: pending.extend((child, depth + 1) for child in item) elif type(item) is dict: pending.extend((child, depth + 1) for child in item.values()) + elif type(item) is float: + # ``parse_constant`` only fires on the NaN/Infinity literals; an + # overflowing number literal such as 1e400 reaches here as inf. + require( + math.isfinite(item), + "Graph evidence: evidence input requires finite JSON numbers.", + ) def read_bytes(path: Path) -> bytes: diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py index 1baba63d3f..3cbd58fd86 100644 --- a/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_evidence.py @@ -665,10 +665,46 @@ def test_item_heavy_native_evidence_is_read_hashed_and_copied(fresh, tmp_path): assert (saved.parent / url).read_bytes() == payload +def test_item_heavy_manifest_round_trips_through_run_evidence(fresh, tmp_path): + """The second #1187 failure: save, load and publish an item-heavy manifest.""" + from dataclasses import replace + + name, receipt = next(iter(fresh.manifest.nodes.items())) + heavy_manifest = replace( + fresh.manifest, + nodes={ + **fresh.manifest.nodes, + name: replace(receipt, receipt={"receipts": [[0, 1]] * 1_500_000}), + }, + ) + index = save_run_evidence( + fresh.compiled, + heavy_manifest, + store=fresh.store, + directory=tmp_path / "attempt-1", + attempt_id="attempt-1", + phase="numerical", + ) + written = next((tmp_path / "attempt-1").rglob("manifest.json")) + assert written.stat().st_size > 10 * 1024 * 1024 + runs = load_run_evidence(index, store=fresh.store) + assert len(runs[0].manifest.nodes[name].receipt["receipts"]) == 1_500_000 + published = publish_run_evidence(index, tmp_path / "out") + entry = json.loads(published.read_bytes())["runs"][0] + copied = published.parent / entry["manifest"]["path"] + assert copied.read_bytes() == written.read_bytes() + republished = load_run_evidence(published, store=fresh.store) + assert ( + republished[0].manifest.nodes[name].receipt["receipts"][0] + == runs[0].manifest.nodes[name].receipt["receipts"][0] + ) + + @pytest.mark.parametrize( "text, message", [ ("[NaN]", "finite"), + ("[1e400]", "finite"), ("[" * 70 + "]" * 70, "nesting"), ('{"a": 1, "a": 2}', "duplicate"), ],