Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,11 @@ add_python_test(notice_covers_deps ${CMAKE_SOURCE_DIR}/tools/ci/notice_covers_de
# reviews: a PR that carries another PR's files lands two merge decisions as one, and one whose work base
# already has reads as a fresh duplicate. Both are invisible to a diffstat; this compares blobs. Device-free.
add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check.py --self-test)
# engine#2021 / engine#2369 review: `registry_build.py --check-gaps` iterates the classes that are PRESENT, so a
# branch that deletes a reviewed class passes it -- measured on #2021, where 41 classes became 40 and --check-gaps
# still printed "none of the 41 reviewed significant classes is mapped". A test row for the three arms that do see it.
add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tests/registry_loss_check_test.py)

add_python_test(hyperloom_benchmark_test ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py)
# engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail.
# `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and
Expand Down
116 changes: 116 additions & 0 deletions tests/registry_loss_check_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""registry_loss_check_test.py -- the three arms, each proven on a real pair of commits.

Each arm builds a throwaway repository, so the check is exercised against git rather than against a mock, and the
failing materials are the ones it is meant to catch: a class removed from `registry/significant.json`, a
`coverage` counter removed from `registry/census.json`, and a class that a `tests/` file stops mentioning.
"""

import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest

CHECK = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools", "ci",
"registry_loss_check.py")


def git(cwd, *args):
return subprocess.run(("git",) + args, cwd=cwd, capture_output=True, text=True,
env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"})


def commit(cwd, message):
git(cwd, "add", "-A")
assert git(cwd, "commit", "-q", "-m", message).returncode == 0


class RegistryLossCheckTest(unittest.TestCase):
def setUp(self):
self.repo = tempfile.mkdtemp(prefix="rlc-")
git(self.repo, "init", "-q", "-b", "main")
os.makedirs(os.path.join(self.repo, "registry"))
os.makedirs(os.path.join(self.repo, "tests"))
self.write("registry/significant.json",
json.dumps({"classes": {"Skylar2ForCausalLM": {"model_type": "skylar2"},
"OmoikaneForCausalLM": {"model_type": "omoikane"}}}, indent=1))
self.write("registry/census.json",
json.dumps({"date": "2026-11-13",
"coverage": {"models": 437432, "with_architecture": 344820},
"raw": {"counts": {"Skylar2ForCausalLM": 1}}}, indent=1))
self.write("tests/census_watch_test.py", "SKYLAR = 'Skylar2ForCausalLM'\n")
commit(self.repo, "base")
self.base = git(self.repo, "rev-parse", "HEAD").stdout.strip()

def tearDown(self):
shutil.rmtree(self.repo, ignore_errors=True)

def write(self, rel, text):
path = os.path.join(self.repo, rel)
os.makedirs(os.path.dirname(path), exist_ok=True) if os.path.dirname(path) else None
with open(path, "w", encoding="utf-8") as handle:
handle.write(text)

def run_check(self, base, head):
return subprocess.run((sys.executable, CHECK, "--base", base, "--head", head), cwd=self.repo,
capture_output=True, text=True)

def test_a_class_that_the_head_lacks_fails_and_names_it(self):
self.write("registry/significant.json", json.dumps({"classes": {"OmoikaneForCausalLM": {}}}, indent=1))
commit(self.repo, "drop a class")
result = self.run_check(self.base, "HEAD")
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertIn("class 'Skylar2ForCausalLM' is on the base and not at the head", result.stdout)

def test_a_coverage_counter_that_the_head_lacks_fails(self):
census = json.loads(open(os.path.join(self.repo, "registry/census.json"), encoding="utf-8").read())
del census["coverage"]["models"]
self.write("registry/census.json", json.dumps(census, indent=1))
commit(self.repo, "drop a counter")
result = self.run_check(self.base, "HEAD")
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertIn("census counter coverage/models (437432) is gone", result.stdout)

def test_a_class_no_test_mentions_any_more_fails(self):
self.write("registry/significant.json", json.dumps({"classes": {"OmoikaneForCausalLM": {}}}, indent=1))
self.write("tests/census_watch_test.py", "SKYLAR = ''\n")
commit(self.repo, "drop the class and its test reference")
result = self.run_check(self.base, "HEAD")
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertIn("Skylar2ForCausalLM: 1 mention(s) under tests/ on the base, 0 at the head", result.stdout)

def test_a_census_value_that_moved_is_not_a_failure(self):
# The census is rebuilt, so a branch carrying a different number has not lost anything.
census = json.loads(open(os.path.join(self.repo, "registry/census.json"), encoding="utf-8").read())
census["coverage"]["models"] = 400001
self.write("registry/census.json", json.dumps(census, indent=1))
commit(self.repo, "older census")
result = self.run_check(self.base, "HEAD")
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)

def test_a_ref_that_cannot_be_read_refuses_instead_of_passing(self):
result = self.run_check(self.base, "no-such-ref")
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
self.assertIn("cannot be read - cannot tell, and that is not a pass", result.stderr)


if __name__ == "__main__":
unittest.main()
200 changes: 200 additions & 0 deletions tools/ci/registry_loss_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""registry_loss_check.py -- did this branch drop a registry entry the base still has?

The registry files are append-mostly: a class in `registry/significant.json` is a reviewed decision, and a
counter in `registry/census.json` is a measurement. Neither is ever meant to lose a row because a branch was
cut before the row landed. `tools/registry_build.py --check-gaps` cannot see that: it iterates the classes
that are *present* and reports "none of the 41 reviewed significant classes is mapped without a reason",
which is equally true of a file with 41 classes and one with 40.

Measured on engine#2021 (`ac36bd0`): main has 23 `*ForCausalLM` classes and the branch 22, `Skylar2` is gone
from both `registry/significant.json` (2 mentions -> 0) and `tests/census_watch_test.py` (5 -> 0), and
`--check-gaps` passes on it. Measured on engine#2369 (`bc4126c`): `registry/census.json` loses `models`,
`with_architecture`, `declined_models` and `architectures_seen`.

Three checks, because the loss shows up in three places:
classes -- a key of `classes` in significant.json that the base has and the head does not
counters -- a top-level counter in census.json that the base has and the head drops or decreases
mentions -- a class name mentioned in `tests/` on the base and not at all in the head

Exit 0 quiet, 1 loss found, 2 a ref could not be read (which is not a pass).
"""

import argparse
import json
import subprocess
import sys

SIGNIFICANT = "registry/significant.json"
CENSUS = "registry/census.json"


def git(*args):
return subprocess.run(("git",) + args, capture_output=True, text=True)


def read_ref(rev, path):
"""Return file text at rev, or None when the rev or the path does not exist there."""
if not rev:
try:
with open(path, encoding="utf-8") as handle:
return handle.read()
except OSError:
return None
shown = git("show", f"{rev}:{path}")
return shown.stdout if shown.returncode == 0 else None


def ref_exists(rev):
if not rev:
return True
return git("rev-parse", "--verify", "--quiet", f"{rev}^{{commit}}").returncode == 0


def classes_of(text, problems, label):
if text is None:
return None
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
problems.append(f"{label}: {SIGNIFICANT} is not valid JSON ({exc})")
return None
classes = data.get("classes")
if not isinstance(classes, dict):
problems.append(f"{label}: {SIGNIFICANT} has no 'classes' object")
return None
return set(classes)


def counters_of(text, problems, label):
"""Every numeric leaf of census.json, flattened, because the counters live under `coverage` and `raw`."""
if text is None:
return None
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
problems.append(f"{label}: {CENSUS} is not valid JSON ({exc})")
return None
flat = {}

def walk(prefix, node):
if isinstance(node, dict):
for key, value in node.items():
walk(f"{prefix}/{key}" if prefix else key, value)
elif isinstance(node, (int, float)) and not isinstance(node, bool):
flat[prefix] = node

walk("", data)
# Only the `coverage` summary's key set is stable: `raw/counts/<Arch>` grows as the census observes more
# architectures, so a branch cut before the last rebuild is missing those keys without having lost anything.
return {key: value for key, value in flat.items() if key.startswith("coverage/")}


def test_mentions(rev, names):
"""How many times each name appears anywhere under tests/ at rev."""
counts = {name: 0 for name in names}
if not names:
return counts
if rev:
files = git("ls-tree", "-r", "--name-only", rev, "tests/").stdout.split()
else:
files = git("ls-files", "tests/").stdout.split()
for path in files:
if not path.endswith((".py", ".sh", ".md", ".txt", ".json")):
continue
text = read_ref(rev, path)
if not text:
continue
for name in names:
counts[name] += text.count(name)
return counts


def main():
parser = argparse.ArgumentParser(description="Refuse a branch that drops registry rows the base has.")
parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)")
parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)")
parser.add_argument("--quiet", action="store_true", help="only print failures")
args = parser.parse_args()

problems = []
for label, rev in (("base", args.base), ("head", args.head)):
if not ref_exists(rev):
problems.append(f"{label} ref '{rev}' cannot be read - cannot tell, and that is not a pass")
if problems:
print("registry_loss_check: " + "; ".join(problems), file=sys.stderr)
return 2

base_classes = classes_of(read_ref(args.base, SIGNIFICANT), problems, "base")
head_classes = classes_of(read_ref(args.head, SIGNIFICANT), problems, "head")
if problems and (base_classes is None or head_classes is None):
print("registry_loss_check: " + "; ".join(problems), file=sys.stderr)
return 2
if base_classes is None or head_classes is None:
print(f"registry_loss_check: {SIGNIFICANT} missing on one side - cannot tell, and that is not a pass",
file=sys.stderr)
return 2

lost_classes = sorted(base_classes - head_classes)
added_classes = sorted(head_classes - base_classes)

base_census = counters_of(read_ref(args.base, CENSUS), problems, "base")
head_census = counters_of(read_ref(args.head, CENSUS), problems, "head")
lost_counters = []
moved_counters = []
if base_census is not None and head_census is not None:
for key, value in sorted(base_census.items()):
if key not in head_census:
lost_counters.append(f"{key} ({value}) is gone")
elif head_census[key] != value:
# Not a failure: the census is a measurement, rebuilt daily, so a branch cut before the last
# rebuild legitimately carries older numbers. A census counter without its date is not
# comparable to one with it, so a change is reported and only an absent counter fails.
moved_counters.append(f"{key} {value} -> {head_census[key]}")

lost_mentions = []
if lost_classes:
base_counts = test_mentions(args.base, lost_classes)
head_counts = test_mentions(args.head, lost_classes)
for name in lost_classes:
if base_counts[name] and not head_counts[name]:
lost_mentions.append(f"{name}: {base_counts[name]} mention(s) under tests/ on the base, 0 at the head")

if not args.quiet:
print(f"registry_loss_check: base {args.base} has {len(base_classes)} class(es), head {args.head} has "
f"{len(head_classes)}")

for item in lost_classes:
print(f"FAIL registry_loss_check: class '{item}' is on the base and not at the head")
for item in lost_counters:
print(f"FAIL registry_loss_check: census counter {item}")
for item in lost_mentions:
print(f"FAIL registry_loss_check: {item}")
if added_classes and not args.quiet:
print(f"registry_loss_check: {len(added_classes)} class(es) added: {', '.join(added_classes)}")
if moved_counters and not args.quiet:
print(f"registry_loss_check: {len(moved_counters)} census counter(s) changed (not a failure; the census is "
f"rebuilt, so check the date at both refs if it matters)")

found = bool(lost_classes or lost_counters or lost_mentions)
if not found and not args.quiet:
print("registry_loss_check: no class, counter or test reference lost")
return 1 if found else 0


if __name__ == "__main__":
sys.exit(main())
Loading