Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 94 additions & 3 deletions tools/ci/tests_referenced_check.py
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
reads as green for ever -- the same defect as #1595, one file kind over.

need = every tests/*.sh
AND every tests/*_test.py that no RUNNER names and that no other test imports (engine#2018)
AND every tools/**/*.py that declares a --self-test (engine#1901): a tool that ships one is claiming it can be
verified without its real inputs, so an unregistered one is an arm nobody runs. For those the corpus is
RUNNERS only (CMakeLists.txt, cmake/, .github/workflows/) -- a mention in another tool's source is a citation,
Expand Down Expand Up @@ -51,6 +52,11 @@
import sys

ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt")
# engine#2018: the existing debt, as a SET baseline rather than 35 allow-list entries with invented reasons -- the same
# shape as tools/ci/add_test_gaps.txt (#1595): the check fails when a NEW unregistered test appears, and reports an entry
# that is no longer needed so the list shrinks. A per-file allow-list of "not a row yet" would be 35 lines nobody can
# tell apart from "forgotten".
PYBASE = pathlib.Path("tools/ci/unreferenced_py_tests.txt")
SCAN = ("", "cmake", "scripts", "tools", "tests", ".github/workflows")
SUFFIXES = {".txt", ".cmake", ".yml", ".yaml", ".sh", ".py"}

Expand All @@ -73,6 +79,19 @@ def allow_list(root):
return out


def py_baseline(root):
"""{basename} from the python-test baseline; comments and blanks skipped."""
path = root / PYBASE
out = set()
if not path.is_file():
return out
for line in path.read_text(errors="replace").splitlines():
line = line.split("#")[0].strip()
if line:
out.add(pathlib.Path(line).name)
return out


def scan_files(root):
"""Every file that can reference a test, as (path, text) -- docs and the allow-list excluded."""
for sub in SCAN:
Expand Down Expand Up @@ -135,6 +154,55 @@ def unregistered_selftest_tools(root):
return drift


def unregistered_py_tests(root):
"""tests/*_test.py that no RUNNER names, excluding files another test imports (engine#2018).

The corpus is RUNNERS, not "any mention": a test importing another test is a citation, the same distinction the
tools rule makes. The `*_test.py` name plus "not imported by a sibling" is what separates a test from a helper
without guessing -- a helper a test imports is called, and a test nothing calls is dead.
"""
tests = root / "tests"
if not tests.is_dir():
return []
cand = sorted(tests.glob("*_test.py"))
if not cand:
return []
imported = set()
for p in list(tests.glob("*.py")) + sorted((root / "tools").glob("*.py")) if (root / "tools").is_dir() else list(tests.glob("*.py")):
try:
text = p.read_text(errors="replace")
except OSError:
continue
for q in cand:
if p == q:
continue
stem = q.stem
if re.search(r"^\s*(?:import\s+%s\b|from\s+%s\s+import)" % (re.escape(stem), re.escape(stem)),
text, re.M):
imported.add(q.name)
runner_text = []
for sub in RUNNERS:
base = root / sub
# RUNNERS holds one FILE (CMakeLists.txt) and two directories; testing is_dir() on all three made every
# cmake registration invisible to this rule, which the self-test arm "a cleared baseline entry is reported"
# caught -- registering a python test in CMakeLists.txt did not clear its baseline line.
paths = [base] if base.is_file() else (
[p for p in sorted(base.rglob("*")) if p.is_file() and p.suffix in {".txt", ".cmake", ".yml", ".yaml", ".py", ".sh"}]
if base.is_dir() else [])
for path in paths:
try:
runner_text.append(path.read_text(errors="replace"))
except OSError:
continue
out = []
for q in cand:
if q.name in imported:
continue
if not any(q.name in t for t in runner_text):
out.append(str(q.relative_to(root)))
return out


def check(root):
tests = root / "tests"
if not tests.is_dir():
Expand All @@ -154,15 +222,25 @@ def check(root):
for name in idle:
print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing"
% (name, ALLOW))
# engine#2018: a SET baseline, not an allow-list -- fail on a NEW one, report one that cleared.
py_now = {pathlib.Path(n).name: n for n in unregistered_py_tests(root)}
py_base = py_baseline(root)
py_drift = sorted(rel for name, rel in py_now.items() if name not in py_base)
py_cleared = sorted(n for n in py_base if n not in py_now)
for rel in py_drift:
print("tests_referenced: %s is a test that no runner names -- add a ctest row (add_python_test in "
"CMakeLists.txt or cmake/*.cmake), or add it to %s if it cannot be one yet" % (rel, PYBASE))
for name in py_cleared:
print("tests_referenced: %s is in %s and no longer needs to be -- drop the line" % (name, PYBASE))
tools_drift = unregistered_selftest_tools(root)
for rel in tools_drift:
print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in "
"CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel)
if drift or gone or idle or tools_drift:
if drift or gone or idle or py_drift or tools_drift:
return 1
print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % (
print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) and %d python test(s) are registered" % (
len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))),
len(_selftest_tools(root))))
len(_selftest_tools(root)), len(list((root / "tests").glob("*_test.py"))) - len(unregistered_py_tests(root))))
return 0


Expand Down Expand Up @@ -244,6 +322,19 @@ def root_with(files):
str(ALLOW): "tests/keeper2.sh # path form is accepted\n"})
cases.append(("a path-form allow entry is accepted", check(r), 0, None))

# 9a. engine#2018: a NEW unregistered python test fails, and a test another test IMPORTS does not count
r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n",
"tests/fresh_test.py": "def f(): pass\n"})
cases.append(("an unregistered tests/*_test.py is drift", check(r), 1, "fresh_test.py"))
cases_root["an unregistered tests/*_test.py is drift"] = r

# 9b. the helper discriminator: imported by a sibling test -> not a test that needs a runner
r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n",
"tests/helper_test.py": "def h(): pass\n",
"tests/caller_test.py": "from helper_test import h\nh()\n",
"tools/ci/unreferenced_py_tests.txt": "tests/caller_test.py\n"})
cases.append(("a test another test imports is not drift", check(r), 0, None))

# 8. no tests/ at all -> cannot tell, not "clean"
r = root_with({"CMakeLists.txt": ""})
cases.append(("no tests/ directory is 'cannot tell'", check(r), 2, None))
Expand Down
36 changes: 36 additions & 0 deletions tools/ci/unreferenced_py_tests.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# tests/*.sh that are deliberately NOT a ctest row, one basename per line, with the reason it must stay out.
# `tools/ci/tests_referenced_check.py` fails when a test is referenced by nothing that can run it, and reports a
# line here that is no longer needed so this list can shrink. The bar for an entry is the same one #1806 used:
# a ctest row has to be ABLE TO FAIL. A script that always exits 0 is an operator measurement, not a test.
#
#
# tests/*_test.py that no ctest row and no workflow step names, measured on main 2026-11-10 (engine#2018).
#
# A SET baseline, not an allow-list: `tools/ci/tests_referenced_check.py` FAILS when a name appears here that is not in
# the tree's current set, and REPORTS a line whose file is registered now, so this can only shrink.
#
# The number is 2, and getting it right took a correction worth recording: the first version of the rule scanned
# `RUNNERS`, which contains one FILE (CMakeLists.txt) and two directories, and tested `is_dir()` on all three -- so every
# `add_python_test(...)` registration in CMakeLists.txt was invisible and 35 of 42 tests looked unregistered. The arm
# "a cleared baseline entry is reported" caught it: registering a test in CMakeLists.txt did not clear its line. The true
# figure is 2 of 42, which is a healthy tree; the rule is here to keep it that way rather than to record a backlog.
#
# Each line is a real test whose runner was never added; the fix per file is one `add_python_test(...)`.
#
tests/hyperloom_benchmark_test.py
tests/moe_expert_chunks_accum_test.py
2 changes: 2 additions & 0 deletions tools/copyright.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
# engine#1820: the other sweep's allow-list, same shape and the same reason -- `#`-comment data, `.txt` has no
# style, and the header it wants is already at the top of the file.
"unreferenced_tests.txt",
# engine#2018: the third test-registration baseline, same shape and the same reason -- data, `.txt` has no style.
"unreferenced_py_tests.txt",
"21serial-console", "90onboard-limits", "Dockerfile", "*.Dockerfile",
# cycle 17: Puppy's switch that skips its first-run Quick Setup is an empty marker file whose only content
# is this notice - Puppy tests for its existence, so a comment is free
Expand Down
Loading