diff --git a/CMakeLists.txt b/CMakeLists.txt index f468d6b07..7698ac6bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -275,6 +275,13 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test. # two files, no build and no submodules; --self-test drives it. add_test(NAME ci_targets COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_test(NAME ci_targets_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +# engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail. +# `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and +# tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink. +add_test(NAME tests_referenced COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_test(NAME tests_referenced_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --self-test) +# engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820). +add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh) # engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three # orphans of that shape landed in two days (51 GiB of GTT for 14 h; an NPU server 70 min past a 6-min ticket; # then two lanes each starting an HRX server outside the token within one half hour). The ancestry test is the diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py new file mode 100755 index 000000000..93df3264f --- /dev/null +++ b/tools/ci/tests_referenced_check.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Every tests/*.sh is mentioned by something that can run it (engine#1820). + +`ci_targets_check.py` asks this question about *targets* -- a test whose executable CI never builds. It cannot +see the other half: a standalone shell test that nothing references at all. Three of those turned up on 2026-11-10 +in one night (#1806 registered one that had no runner; #1777 added one nothing referenced; #1809 added one whose +target ci.yml never builds, which is the case ci_targets_check does cover). A test nobody runs cannot fail, so it +reads as green for ever -- the same defect as #1595, one file kind over. + + need = every tests/*.sh + have = every file that MENTIONS its basename, in a place that can run things: + CMakeLists.txt, cmake/**, scripts/**, tools/**, tests/**, .github/workflows/** + drift = need - have - allow-list + +A mention inside the file itself does not count: a test's own header or usage line naming itself is not a runner, +and counting it would make the whole check vacuous (measured on this tree: it is the one mistake the hand-written +loop made first). + +Documentation is deliberately NOT a reference: a sentence in a `.md` that names a test is not something that runs it. + +The allow-list is `tools/ci/unreferenced_tests.txt`, one basename per line with an optional `# reason` -- the same +shape as `add_test_gaps.txt`. An entry that no longer needs to be there is reported so the list can shrink, which +keeps it honest rather than accumulating. + +usage: tools/ci/tests_referenced_check.py [--check] [--self-test] [--root DIR] +Exit: 0 every test is reachable, 1 an unreferenced one (named), 2 cannot tell (no tests/ directory). +""" + +import argparse +import pathlib +import sys + +ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt") +SCAN = ("", "cmake", "scripts", "tools", "tests", ".github/workflows") +SUFFIXES = {".txt", ".cmake", ".yml", ".yaml", ".sh", ".py"} + + +def allow_list(root): + """{basename: reason} from the allow-list; comments and blanks skipped.""" + path = root / ALLOW + out = {} + if not path.is_file(): + return out + for line in path.read_text(errors="replace").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + name, _, reason = line.partition("#") + out[name.strip()] = reason.strip() + return out + + +def scan_files(root): + """Every file that can reference a test, as (path, text) -- docs and the allow-list excluded.""" + for sub in SCAN: + base = root / sub if sub else root + if not base.is_dir(): + continue + pattern = "*" if sub else "CMakeLists.txt" + for path in sorted(base.glob(pattern)): + if path.is_dir() or path.suffix not in SUFFIXES: + continue + if path == root / ALLOW: + continue # the allow-list names the file too; it is not a runner + try: + yield path, path.read_text(errors="replace") + except OSError: + continue + + +def referenced(root, need): + """{name: [files]}, ignoring a file mentioning its own basename.""" + found = {name: [] for name in need} + for path, text in scan_files(root): + for name in need: + if name not in text: + continue + if path.name == name: + continue # a test naming itself is not a runner + found[name].append(str(path.relative_to(root))) + return found + + +def check(root): + tests = root / "tests" + if not tests.is_dir(): + print("tests_referenced: cannot tell: no tests/ directory under %s" % root, file=sys.stderr) + return 2 + need = sorted(p.name for p in tests.glob("*.sh")) + allow = allow_list(root) + found = referenced(root, need) + drift = [n for n in need if not found[n] and n not in allow] + stale = [n for n in allow if n not in need] + for name in drift: + print("tests_referenced: %s is referenced by nothing that can run it -- register it (add_test in " + "CMakeLists.txt or cmake/*.cmake), or list it in %s with the reason it must not be a row" % (name, ALLOW)) + for name in stale: + print("tests_referenced: %s is in %s but no such test exists any more -- drop the line" % (name, ALLOW)) + if drift: + return 1 + print("tests_referenced: all %d test script(s) are referenced%s" % ( + len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))))) + return 0 + + +def self_test(): + """Synthetic roots: a check that only ever sees a good tree proves nothing.""" + import tempfile + + cases = [] + + def root_with(files): + d = pathlib.Path(tempfile.mkdtemp()) + for rel, body in files.items(): + p = d / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body) + return d + + # 1. a test nobody mentions -> drift, and it is NAMED + r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"}) + cases_root = {} + cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r + + # 2. referenced from CMakeLists.txt -> clean + r = root_with({"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n"}) + cases.append(("a registration in CMakeLists.txt counts", check(r), 0, None)) + + # 3. referenced from an included family file -> clean + r = root_with({"tests/fam.sh": "#!/bin/sh\n", "CMakeLists.txt": "", + "cmake/tests-family.cmake": "add_test(NAME fam COMMAND bash tests/fam.sh)\n"}) + cases.append(("a registration in cmake/*.cmake counts", check(r), 0, None)) + + # 4. referenced from a workflow -> clean + r = root_with({"tests/ci.sh": "#!/bin/sh\n", "CMakeLists.txt": "", + ".github/workflows/ci.yml": "run: bash tests/ci.sh\n"}) + cases.append(("a mention in a workflow counts", check(r), 0, None)) + + # 5. a test naming ONLY itself -> drift (the vacuous case) + r = root_with({"tests/self.sh": "#!/bin/sh\n# usage: tests/self.sh\n", "CMakeLists.txt": ""}) + cases.append(("a test naming itself is not a runner", check(r), 1, "self.sh")); cases_root["a test naming itself is not a runner"] = r + + # 6. allow-listed with a reason -> clean, and the reason is carried + r = root_with({"tests/keeper.sh": "#!/bin/sh\n", "CMakeLists.txt": "", + str(ALLOW): "keeper.sh # an operator measurement: always exits 0, so a row would be green noise\n"}) + cases.append(("an allow-listed test is not drift", check(r), 0, None)) + + # 7. a stale allow entry -> reported, but not a failure + r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "tests/one.sh\n", + str(ALLOW): "gone.sh # removed long ago\n"}) + cases.append(("a stale allow entry is progress, not drift", check(r), 0, "gone.sh")); cases_root["a stale allow entry is progress, not drift"] = r + + # 8. no tests/ at all -> cannot tell, not "clean" + r = root_with({"CMakeLists.txt": ""}) + cases.append(("no tests/ directory is 'cannot tell'", check(r), 2, None)) + + failures = 0 + import contextlib + import io + for what, got, want, mentions in cases: + if got != want: + print("FAIL %s: rc %s, wanted %s" % (what, got, want)) + failures += 1 + continue + if mentions: + # re-run capturing the output: the verdict must NAME the offending test, or a reader cannot act on it + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + if what.startswith("a stale"): + check(cases_root[what]) + else: + check(cases_root[what]) + if mentions not in buf.getvalue(): + print("FAIL %s: rc was right but the output does not name %s" % (what, mentions)) + failures += 1 + continue + print("ok %s" % what) + print("tests_referenced self-test: %d failure(s)" % failures) + return 1 if failures else 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--check", action="store_true") + ap.add_argument("--self-test", action="store_true") + ap.add_argument("--root", default=".", help="tree to inspect (default: the current directory)") + a = ap.parse_args() + if a.self_test: + return self_test() + return check(pathlib.Path(a.root).resolve()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci/unreferenced_tests.txt b/tools/ci/unreferenced_tests.txt new file mode 100644 index 000000000..be82343ef --- /dev/null +++ b/tools/ci/unreferenced_tests.txt @@ -0,0 +1,21 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/*.sh that are deliberately NOT a ctest row, one basename per line, with the reason it must stay out. +# `tools/ci/tests_referenced_check.py` fails when a test is referenced by nothing that can run it, and reports a +# line here that is no longer needed so this list can shrink. The bar for an entry is the same one #1806 used: +# a ctest row has to be ABLE TO FAIL. A script that always exits 0 is an operator measurement, not a test. +# +anchor_rerun.sh # an operator measurement: needs real GGUFs and an output dir, allows 30 min per model, prints PASS/FAIL/SKIP per model and ALWAYS exits 0 -- as a ctest row it would either skip for ever or report green over a FAIL line (engine#1800) diff --git a/tools/copyright.py b/tools/copyright.py index 561de0287..d8db59bd5 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -41,6 +41,9 @@ # engine#1595: the direction-B ratchet's baseline is a '#'-comment data file too, and `.txt` has no style of # its own (a blanket "*.txt" would demand a notice in files that are data, not code). "add_test_gaps.txt", + # engine#1820: the other sweep's allow-list, same shape and the same reason -- `#`-comment data, `.txt` has no + # style, and the header it wants is already at the top of the file. + "unreferenced_tests.txt", "21serial-console", "90onboard-limits", "Dockerfile", "*.Dockerfile", # cycle 17: Puppy's switch that skips its first-run Quick Setup is an empty marker file whose only content # is this notice - Puppy tests for its existence, so a comment is free