From 00a368d566d30eb771ba57243dbd8c34f66d097b Mon Sep 17 00:00:00 2001 From: pi-prs2 Date: Sun, 15 Nov 2026 09:15:04 -0400 Subject: [PATCH] tests: the census summary must be what its own raw block implies registry/census.json carries both the raw sweep and the summary derived from it (tools/census.py:coverage), so the summary can be stale or hand-edited while the raw data is fine: main shipped `declined_models: 430` over a raw block that computes to 434, and nothing noticed until a regeneration rewrote the file. The check re-derives the summary from the committed raw block and compares every field, so a drift names both values: 'declined_models: (434, 430)'. Arms: the committed file passes; the same file with declined_models set back to 430 fails by name (engine#2533 regenerated it, this is the arm that would have caught it). Refs #2528 --- tests/census_watch_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/census_watch_test.py b/tests/census_watch_test.py index bbe02b20f..23c3527c5 100644 --- a/tests/census_watch_test.py +++ b/tests/census_watch_test.py @@ -40,6 +40,7 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(ROOT, "tools")) import census_watch as cw # noqa: E402 +import census # noqa: E402 (coverage(): the committed summary is derived, so it can be re-derived) ARCHS = { "Qwen3ForCausalLM": {"gguf": "qwen3", "backends": ["hrx"]}, @@ -1113,6 +1114,15 @@ def run_row(rid, arch, q4nx=True): check(f"flm answers: every answered architecture has a row in docs/npu.md (missing: {_undocumented})", not _undocumented) +# The summary is derived from the raw block, so the only proof it is right is re-deriving it. A stale or +# hand-edited count reads as coverage we do not have: main shipped `declined_models: 430` over raw data +# that computes to 434 (engine#2533 fixed that file; this is the arm that would have caught it). +_census = json.load(open(os.path.join(ROOT, "registry", "census.json"))) +_derived = census.coverage(_census["raw"]) +_drift = {k: (v, _census["coverage"].get(k)) for k, v in _derived.items() + if (round(v, 2) if isinstance(v, float) else v) != _census["coverage"].get(k)} +check(f"census: the committed summary is what its own raw block implies (drift: {_drift})", not _drift) + # 21b. engine#2171: a pin that names its `paths` moves only when a commit touches one of them. # vllm-project/semantic-router lands dozens of router-side commits a day; the d3 contract it is pinned for # did not change between a9219240e and 15b4fb5ad, yet the whole-repo head alerted on every run.