diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py old mode 100755 new mode 100644 index 2df91ec28..8dd2f3818 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -22,6 +22,7 @@ reads as green for ever -- the same defect as #1595, one file kind over. need = every tests/*.sh + AND every tests/*_test.py that no RUNNER names and that no other test imports (engine#2018) AND every tools/**/*.py that declares a --self-test (engine#1901): a tool that ships one is claiming it can be verified without its real inputs, so an unregistered one is an arm nobody runs. For those the corpus is RUNNERS only (CMakeLists.txt, cmake/, .github/workflows/) -- a mention in another tool's source is a citation, @@ -51,6 +52,11 @@ import sys ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt") +# engine#2018: the existing debt, as a SET baseline rather than 35 allow-list entries with invented reasons -- the same +# shape as tools/ci/add_test_gaps.txt (#1595): the check fails when a NEW unregistered test appears, and reports an entry +# that is no longer needed so the list shrinks. A per-file allow-list of "not a row yet" would be 35 lines nobody can +# tell apart from "forgotten". +PYBASE = pathlib.Path("tools/ci/unreferenced_py_tests.txt") SCAN = ("", "cmake", "scripts", "tools", "tests", ".github/workflows") SUFFIXES = {".txt", ".cmake", ".yml", ".yaml", ".sh", ".py"} @@ -73,6 +79,19 @@ def allow_list(root): return out +def py_baseline(root): + """{basename} from the python-test baseline; comments and blanks skipped.""" + path = root / PYBASE + out = set() + if not path.is_file(): + return out + for line in path.read_text(errors="replace").splitlines(): + line = line.split("#")[0].strip() + if line: + out.add(pathlib.Path(line).name) + return out + + def scan_files(root): """Every file that can reference a test, as (path, text) -- docs and the allow-list excluded.""" for sub in SCAN: @@ -135,6 +154,55 @@ def unregistered_selftest_tools(root): return drift +def unregistered_py_tests(root): + """tests/*_test.py that no RUNNER names, excluding files another test imports (engine#2018). + + The corpus is RUNNERS, not "any mention": a test importing another test is a citation, the same distinction the + tools rule makes. The `*_test.py` name plus "not imported by a sibling" is what separates a test from a helper + without guessing -- a helper a test imports is called, and a test nothing calls is dead. + """ + tests = root / "tests" + if not tests.is_dir(): + return [] + cand = sorted(tests.glob("*_test.py")) + if not cand: + return [] + imported = set() + for p in list(tests.glob("*.py")) + sorted((root / "tools").glob("*.py")) if (root / "tools").is_dir() else list(tests.glob("*.py")): + try: + text = p.read_text(errors="replace") + except OSError: + continue + for q in cand: + if p == q: + continue + stem = q.stem + if re.search(r"^\s*(?:import\s+%s\b|from\s+%s\s+import)" % (re.escape(stem), re.escape(stem)), + text, re.M): + imported.add(q.name) + runner_text = [] + for sub in RUNNERS: + base = root / sub + # RUNNERS holds one FILE (CMakeLists.txt) and two directories; testing is_dir() on all three made every + # cmake registration invisible to this rule, which the self-test arm "a cleared baseline entry is reported" + # caught -- registering a python test in CMakeLists.txt did not clear its baseline line. + paths = [base] if base.is_file() else ( + [p for p in sorted(base.rglob("*")) if p.is_file() and p.suffix in {".txt", ".cmake", ".yml", ".yaml", ".py", ".sh"}] + if base.is_dir() else []) + for path in paths: + try: + runner_text.append(path.read_text(errors="replace")) + except OSError: + continue + out = [] + for q in cand: + if q.name in imported: + continue + if not any(q.name in t for t in runner_text): + out.append(str(q.relative_to(root))) + return out + + def check(root): tests = root / "tests" if not tests.is_dir(): @@ -154,15 +222,25 @@ def check(root): for name in idle: print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing" % (name, ALLOW)) + # engine#2018: a SET baseline, not an allow-list -- fail on a NEW one, report one that cleared. + py_now = {pathlib.Path(n).name: n for n in unregistered_py_tests(root)} + py_base = py_baseline(root) + py_drift = sorted(rel for name, rel in py_now.items() if name not in py_base) + py_cleared = sorted(n for n in py_base if n not in py_now) + for rel in py_drift: + print("tests_referenced: %s is a test that no runner names -- add a ctest row (add_python_test in " + "CMakeLists.txt or cmake/*.cmake), or add it to %s if it cannot be one yet" % (rel, PYBASE)) + for name in py_cleared: + print("tests_referenced: %s is in %s and no longer needs to be -- drop the line" % (name, PYBASE)) tools_drift = unregistered_selftest_tools(root) for rel in tools_drift: print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in " "CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel) - if drift or gone or idle or tools_drift: + if drift or gone or idle or py_drift or tools_drift: return 1 - print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % ( + print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) and %d python test(s) are registered" % ( len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))), - len(_selftest_tools(root)))) + len(_selftest_tools(root)), len(list((root / "tests").glob("*_test.py"))) - len(unregistered_py_tests(root)))) return 0 @@ -244,6 +322,19 @@ def root_with(files): str(ALLOW): "tests/keeper2.sh # path form is accepted\n"}) cases.append(("a path-form allow entry is accepted", check(r), 0, None)) + # 9a. engine#2018: a NEW unregistered python test fails, and a test another test IMPORTS does not count + r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n", + "tests/fresh_test.py": "def f(): pass\n"}) + cases.append(("an unregistered tests/*_test.py is drift", check(r), 1, "fresh_test.py")) + cases_root["an unregistered tests/*_test.py is drift"] = r + + # 9b. the helper discriminator: imported by a sibling test -> not a test that needs a runner + r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n", + "tests/helper_test.py": "def h(): pass\n", + "tests/caller_test.py": "from helper_test import h\nh()\n", + "tools/ci/unreferenced_py_tests.txt": "tests/caller_test.py\n"}) + cases.append(("a test another test imports is not drift", check(r), 0, None)) + # 8. no tests/ at all -> cannot tell, not "clean" r = root_with({"CMakeLists.txt": ""}) cases.append(("no tests/ directory is 'cannot tell'", check(r), 2, None)) diff --git a/tools/ci/unreferenced_py_tests.txt b/tools/ci/unreferenced_py_tests.txt new file mode 100644 index 000000000..9cef48019 --- /dev/null +++ b/tools/ci/unreferenced_py_tests.txt @@ -0,0 +1,36 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/*.sh that are deliberately NOT a ctest row, one basename per line, with the reason it must stay out. +# `tools/ci/tests_referenced_check.py` fails when a test is referenced by nothing that can run it, and reports a +# line here that is no longer needed so this list can shrink. The bar for an entry is the same one #1806 used: +# a ctest row has to be ABLE TO FAIL. A script that always exits 0 is an operator measurement, not a test. +# +# +# tests/*_test.py that no ctest row and no workflow step names, measured on main 2026-11-10 (engine#2018). +# +# A SET baseline, not an allow-list: `tools/ci/tests_referenced_check.py` FAILS when a name appears here that is not in +# the tree's current set, and REPORTS a line whose file is registered now, so this can only shrink. +# +# The number is 2, and getting it right took a correction worth recording: the first version of the rule scanned +# `RUNNERS`, which contains one FILE (CMakeLists.txt) and two directories, and tested `is_dir()` on all three -- so every +# `add_python_test(...)` registration in CMakeLists.txt was invisible and 35 of 42 tests looked unregistered. The arm +# "a cleared baseline entry is reported" caught it: registering a test in CMakeLists.txt did not clear its line. The true +# figure is 2 of 42, which is a healthy tree; the rule is here to keep it that way rather than to record a backlog. +# +# Each line is a real test whose runner was never added; the fix per file is one `add_python_test(...)`. +# +tests/hyperloom_benchmark_test.py +tests/moe_expert_chunks_accum_test.py diff --git a/tools/copyright.py b/tools/copyright.py index d8db59bd5..840a92fa9 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -44,6 +44,8 @@ # engine#1820: the other sweep's allow-list, same shape and the same reason -- `#`-comment data, `.txt` has no # style, and the header it wants is already at the top of the file. "unreferenced_tests.txt", + # engine#2018: the third test-registration baseline, same shape and the same reason -- data, `.txt` has no style. + "unreferenced_py_tests.txt", "21serial-console", "90onboard-limits", "Dockerfile", "*.Dockerfile", # cycle 17: Puppy's switch that skips its first-run Quick Setup is an empty marker file whose only content # is this notice - Puppy tests for its existence, so a comment is free