From 00616c7b65a5f35f010450c43107c1f802ae8ee5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 18:11:06 -0400 Subject: [PATCH] registry_loss_check: a branch that drops a reviewed class, a coverage counter or a test's reference engine#2021 at ac36bd0 has 41 -> 40 classes and tests/census_watch_test.py mentions Skylar2 5 -> 0 times, and tools/registry_build.py --check-gaps passes on it: that checker iterates the classes that are present, so it reports 'none of the 41 reviewed significant classes is mapped' for a file with 40 of them. Three arms on a git pair (tests/registry_loss_check_test.py, five cases incl. a clean control, a moved census value that must NOT fail, and an unreadable ref that must refuse with rc 2). A coverage counter that is absent fails; one whose value moved does not, because the census is rebuilt and a counter without its date is not comparable. --- CMakeLists.txt | 5 + tests/registry_loss_check_test.py | 116 +++++++++++++++++ tools/ci/registry_loss_check.py | 200 ++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 tests/registry_loss_check_test.py create mode 100644 tools/ci/registry_loss_check.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 98d9feffa..219c06d7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -436,6 +436,11 @@ add_python_test(notice_covers_deps ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_de # reviews: a PR that carries another PR's files lands two merge decisions as one, and one whose work base # already has reads as a fresh duplicate. Both are invisible to a diffstat; this compares blobs. Device-free. add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check.py --self-test) +# engine#2021 / engine#2369 review: `registry_build.py --check-gaps` iterates the classes that are PRESENT, so a +# branch that deletes a reviewed class passes it -- measured on #2021, where 41 classes became 40 and --check-gaps +# still printed "none of the 41 reviewed significant classes is mapped". A test row for the three arms that do see it. +add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tests/registry_loss_check_test.py) + add_python_test(hyperloom_benchmark_test ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py) # engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail. # `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and diff --git a/tests/registry_loss_check_test.py b/tests/registry_loss_check_test.py new file mode 100644 index 000000000..56082fc1b --- /dev/null +++ b/tests/registry_loss_check_test.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""registry_loss_check_test.py -- the three arms, each proven on a real pair of commits. + +Each arm builds a throwaway repository, so the check is exercised against git rather than against a mock, and the +failing materials are the ones it is meant to catch: a class removed from `registry/significant.json`, a +`coverage` counter removed from `registry/census.json`, and a class that a `tests/` file stops mentioning. +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +CHECK = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools", "ci", + "registry_loss_check.py") + + +def git(cwd, *args): + return subprocess.run(("git",) + args, cwd=cwd, capture_output=True, text=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}) + + +def commit(cwd, message): + git(cwd, "add", "-A") + assert git(cwd, "commit", "-q", "-m", message).returncode == 0 + + +class RegistryLossCheckTest(unittest.TestCase): + def setUp(self): + self.repo = tempfile.mkdtemp(prefix="rlc-") + git(self.repo, "init", "-q", "-b", "main") + os.makedirs(os.path.join(self.repo, "registry")) + os.makedirs(os.path.join(self.repo, "tests")) + self.write("registry/significant.json", + json.dumps({"classes": {"Skylar2ForCausalLM": {"model_type": "skylar2"}, + "OmoikaneForCausalLM": {"model_type": "omoikane"}}}, indent=1)) + self.write("registry/census.json", + json.dumps({"date": "2026-11-13", + "coverage": {"models": 437432, "with_architecture": 344820}, + "raw": {"counts": {"Skylar2ForCausalLM": 1}}}, indent=1)) + self.write("tests/census_watch_test.py", "SKYLAR = 'Skylar2ForCausalLM'\n") + commit(self.repo, "base") + self.base = git(self.repo, "rev-parse", "HEAD").stdout.strip() + + def tearDown(self): + shutil.rmtree(self.repo, ignore_errors=True) + + def write(self, rel, text): + path = os.path.join(self.repo, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) if os.path.dirname(path) else None + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + + def run_check(self, base, head): + return subprocess.run((sys.executable, CHECK, "--base", base, "--head", head), cwd=self.repo, + capture_output=True, text=True) + + def test_a_class_that_the_head_lacks_fails_and_names_it(self): + self.write("registry/significant.json", json.dumps({"classes": {"OmoikaneForCausalLM": {}}}, indent=1)) + commit(self.repo, "drop a class") + result = self.run_check(self.base, "HEAD") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("class 'Skylar2ForCausalLM' is on the base and not at the head", result.stdout) + + def test_a_coverage_counter_that_the_head_lacks_fails(self): + census = json.loads(open(os.path.join(self.repo, "registry/census.json"), encoding="utf-8").read()) + del census["coverage"]["models"] + self.write("registry/census.json", json.dumps(census, indent=1)) + commit(self.repo, "drop a counter") + result = self.run_check(self.base, "HEAD") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("census counter coverage/models (437432) is gone", result.stdout) + + def test_a_class_no_test_mentions_any_more_fails(self): + self.write("registry/significant.json", json.dumps({"classes": {"OmoikaneForCausalLM": {}}}, indent=1)) + self.write("tests/census_watch_test.py", "SKYLAR = ''\n") + commit(self.repo, "drop the class and its test reference") + result = self.run_check(self.base, "HEAD") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("Skylar2ForCausalLM: 1 mention(s) under tests/ on the base, 0 at the head", result.stdout) + + def test_a_census_value_that_moved_is_not_a_failure(self): + # The census is rebuilt, so a branch carrying a different number has not lost anything. + census = json.loads(open(os.path.join(self.repo, "registry/census.json"), encoding="utf-8").read()) + census["coverage"]["models"] = 400001 + self.write("registry/census.json", json.dumps(census, indent=1)) + commit(self.repo, "older census") + result = self.run_check(self.base, "HEAD") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_ref_that_cannot_be_read_refuses_instead_of_passing(self): + result = self.run_check(self.base, "no-such-ref") + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("cannot be read - cannot tell, and that is not a pass", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ci/registry_loss_check.py b/tools/ci/registry_loss_check.py new file mode 100644 index 000000000..0b26995b1 --- /dev/null +++ b/tools/ci/registry_loss_check.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""registry_loss_check.py -- did this branch drop a registry entry the base still has? + +The registry files are append-mostly: a class in `registry/significant.json` is a reviewed decision, and a +counter in `registry/census.json` is a measurement. Neither is ever meant to lose a row because a branch was +cut before the row landed. `tools/registry_build.py --check-gaps` cannot see that: it iterates the classes +that are *present* and reports "none of the 41 reviewed significant classes is mapped without a reason", +which is equally true of a file with 41 classes and one with 40. + +Measured on engine#2021 (`ac36bd0`): main has 23 `*ForCausalLM` classes and the branch 22, `Skylar2` is gone +from both `registry/significant.json` (2 mentions -> 0) and `tests/census_watch_test.py` (5 -> 0), and +`--check-gaps` passes on it. Measured on engine#2369 (`bc4126c`): `registry/census.json` loses `models`, +`with_architecture`, `declined_models` and `architectures_seen`. + +Three checks, because the loss shows up in three places: + classes -- a key of `classes` in significant.json that the base has and the head does not + counters -- a top-level counter in census.json that the base has and the head drops or decreases + mentions -- a class name mentioned in `tests/` on the base and not at all in the head + +Exit 0 quiet, 1 loss found, 2 a ref could not be read (which is not a pass). +""" + +import argparse +import json +import subprocess +import sys + +SIGNIFICANT = "registry/significant.json" +CENSUS = "registry/census.json" + + +def git(*args): + return subprocess.run(("git",) + args, capture_output=True, text=True) + + +def read_ref(rev, path): + """Return file text at rev, or None when the rev or the path does not exist there.""" + if not rev: + try: + with open(path, encoding="utf-8") as handle: + return handle.read() + except OSError: + return None + shown = git("show", f"{rev}:{path}") + return shown.stdout if shown.returncode == 0 else None + + +def ref_exists(rev): + if not rev: + return True + return git("rev-parse", "--verify", "--quiet", f"{rev}^{{commit}}").returncode == 0 + + +def classes_of(text, problems, label): + if text is None: + return None + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + problems.append(f"{label}: {SIGNIFICANT} is not valid JSON ({exc})") + return None + classes = data.get("classes") + if not isinstance(classes, dict): + problems.append(f"{label}: {SIGNIFICANT} has no 'classes' object") + return None + return set(classes) + + +def counters_of(text, problems, label): + """Every numeric leaf of census.json, flattened, because the counters live under `coverage` and `raw`.""" + if text is None: + return None + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + problems.append(f"{label}: {CENSUS} is not valid JSON ({exc})") + return None + flat = {} + + def walk(prefix, node): + if isinstance(node, dict): + for key, value in node.items(): + walk(f"{prefix}/{key}" if prefix else key, value) + elif isinstance(node, (int, float)) and not isinstance(node, bool): + flat[prefix] = node + + walk("", data) + # Only the `coverage` summary's key set is stable: `raw/counts/` grows as the census observes more + # architectures, so a branch cut before the last rebuild is missing those keys without having lost anything. + return {key: value for key, value in flat.items() if key.startswith("coverage/")} + + +def test_mentions(rev, names): + """How many times each name appears anywhere under tests/ at rev.""" + counts = {name: 0 for name in names} + if not names: + return counts + if rev: + files = git("ls-tree", "-r", "--name-only", rev, "tests/").stdout.split() + else: + files = git("ls-files", "tests/").stdout.split() + for path in files: + if not path.endswith((".py", ".sh", ".md", ".txt", ".json")): + continue + text = read_ref(rev, path) + if not text: + continue + for name in names: + counts[name] += text.count(name) + return counts + + +def main(): + parser = argparse.ArgumentParser(description="Refuse a branch that drops registry rows the base has.") + parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)") + parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)") + parser.add_argument("--quiet", action="store_true", help="only print failures") + args = parser.parse_args() + + problems = [] + for label, rev in (("base", args.base), ("head", args.head)): + if not ref_exists(rev): + problems.append(f"{label} ref '{rev}' cannot be read - cannot tell, and that is not a pass") + if problems: + print("registry_loss_check: " + "; ".join(problems), file=sys.stderr) + return 2 + + base_classes = classes_of(read_ref(args.base, SIGNIFICANT), problems, "base") + head_classes = classes_of(read_ref(args.head, SIGNIFICANT), problems, "head") + if problems and (base_classes is None or head_classes is None): + print("registry_loss_check: " + "; ".join(problems), file=sys.stderr) + return 2 + if base_classes is None or head_classes is None: + print(f"registry_loss_check: {SIGNIFICANT} missing on one side - cannot tell, and that is not a pass", + file=sys.stderr) + return 2 + + lost_classes = sorted(base_classes - head_classes) + added_classes = sorted(head_classes - base_classes) + + base_census = counters_of(read_ref(args.base, CENSUS), problems, "base") + head_census = counters_of(read_ref(args.head, CENSUS), problems, "head") + lost_counters = [] + moved_counters = [] + if base_census is not None and head_census is not None: + for key, value in sorted(base_census.items()): + if key not in head_census: + lost_counters.append(f"{key} ({value}) is gone") + elif head_census[key] != value: + # Not a failure: the census is a measurement, rebuilt daily, so a branch cut before the last + # rebuild legitimately carries older numbers. A census counter without its date is not + # comparable to one with it, so a change is reported and only an absent counter fails. + moved_counters.append(f"{key} {value} -> {head_census[key]}") + + lost_mentions = [] + if lost_classes: + base_counts = test_mentions(args.base, lost_classes) + head_counts = test_mentions(args.head, lost_classes) + for name in lost_classes: + if base_counts[name] and not head_counts[name]: + lost_mentions.append(f"{name}: {base_counts[name]} mention(s) under tests/ on the base, 0 at the head") + + if not args.quiet: + print(f"registry_loss_check: base {args.base} has {len(base_classes)} class(es), head {args.head} has " + f"{len(head_classes)}") + + for item in lost_classes: + print(f"FAIL registry_loss_check: class '{item}' is on the base and not at the head") + for item in lost_counters: + print(f"FAIL registry_loss_check: census counter {item}") + for item in lost_mentions: + print(f"FAIL registry_loss_check: {item}") + if added_classes and not args.quiet: + print(f"registry_loss_check: {len(added_classes)} class(es) added: {', '.join(added_classes)}") + if moved_counters and not args.quiet: + print(f"registry_loss_check: {len(moved_counters)} census counter(s) changed (not a failure; the census is " + f"rebuilt, so check the date at both refs if it matters)") + + found = bool(lost_classes or lost_counters or lost_mentions) + if not found and not args.quiet: + print("registry_loss_check: no class, counter or test reference lost") + return 1 if found else 0 + + +if __name__ == "__main__": + sys.exit(main())