From 215054350036c3b59b2a73104f61f7770f5d7aa0 Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 10 Nov 2026 02:33:04 -0400 Subject: [PATCH] ci: a tests/*.sh that nothing references is now a failing row (engine#1820) ci_targets_check owns 'a test is declared and never runs' but reasons about targets, and a standalone .sh has none -- three PRs in one night shipped a test nothing ran. Measured on origin/main fafabbd2, the hand-loop returns two entries: tests/anchor_rerun.sh (the deliberate exclusion) and tests/release_check_hrx_init_reason_test.sh, which reached main with #1777. - tools/ci/unreferenced_tests_check.py: need/have/drift, both directions (a stale excuse fails), --self-test over a fixture tree. A comment line naming a test is not a reference: the first version of this checker was blind to release_check_hrx_init_reason_test.sh because its own docstring named it. - tools/ci/unreferenced_tests.txt: the excuse list, reason on the same line. - CMakeLists: register release_check_hrx_init_reason (the drift this finds) and the two check rows. Red/green measured: dangling file -> rc 1; the hrx row removed -> rc 1 naming it; stale excuse -> rc 1; clean tree -> rc 0 (73 tests, 1 excused). Under ctest: 3/3 passed in 2.69 s. --- CMakeLists.txt | 10 ++ tools/ci/unreferenced_tests.txt | 23 +++ tools/ci/unreferenced_tests_check.py | 220 +++++++++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 tools/ci/unreferenced_tests.txt create mode 100755 tools/ci/unreferenced_tests_check.py diff --git a/CMakeLists.txt b/CMakeLists.txt index f468d6b07..e34dc41fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,6 +208,10 @@ add_test(NAME recipes_route COMMAND ${CMAKE_SOURCE_DIR}/tests/recipes_route.sh $ # it is here because nothing else runs it: the rows had 128 assertions and no runner. It names no add_executable, so it # adds nothing to the set of test targets the CI --target list would have to build (the direction-B measurement, #1595). add_test(NAME release_check_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh --self-test) +# engine#1766/#1767: the hrx-device row's init-reason extraction, three arms. Script-based (it reads the row function +# out of os/mini/release-check.sh), no build, no device. It reached main with #1777 referenced by nothing; registering +# it here is what lets the engine#1820 row below pass on a tree that tells the truth. +add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh) # engine#1800: the 1bit OS base layer's mode normaliser and reporter (scripts/pack-elf.sh, engine#1474), run over a # crafted tree. Same shape as the row above: no build, no device, no target, so it adds nothing to the CI --target list. # It was in tests/ with nothing referencing it, so the property it pins (the normaliser only NARROWS modes: a 0600 @@ -275,6 +279,12 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test. # two files, no build and no submodules; --self-test drives it. add_test(NAME ci_targets COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_test(NAME ci_targets_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +# engine#1820: the shell half of the same idea ci_targets owns for targets -- every tests/*.sh must be named by +# something that could run it (CMakeLists, cmake/, .github/, scripts/, tools/, or another test), or be excused with a +# reason in tools/ci/unreferenced_tests.txt. Three PRs in one night shipped a test nothing ran and this had no row; +# #1777's tests/release_check_hrx_init_reason_test.sh is on main today because of it. No target, no device, no network. +add_test(NAME unreferenced_tests COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/unreferenced_tests_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_test(NAME unreferenced_tests_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/unreferenced_tests_check.py --self-test) # engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three # orphans of that shape landed in two days (51 GiB of GTT for 14 h; an NPU server 70 min past a 6-min ticket; # then two lanes each starting an HRX server outside the token within one half hour). The ancestry test is the diff --git a/tools/ci/unreferenced_tests.txt b/tools/ci/unreferenced_tests.txt new file mode 100644 index 000000000..473352a26 --- /dev/null +++ b/tools/ci/unreferenced_tests.txt @@ -0,0 +1,23 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/*.sh that are deliberately referenced by nothing (engine#1820). One path per line, then the reason -- +# the reason is the point: an entry without one fails, and so does an entry whose file is now registered, so this +# list cannot rot into a dumping ground for tests nobody got round to wiring. +# +# Shape: tests/.sh # why it is not a ctest row +# +tests/anchor_rerun.sh # always exits 0 (it is a skip/exit-semantics fixture), so as a ctest row it would either skip forever or report green over a FAIL line +tests/anchor_rerun.sh # stale diff --git a/tools/ci/unreferenced_tests_check.py b/tools/ci/unreferenced_tests_check.py new file mode 100755 index 000000000..30fc6e343 --- /dev/null +++ b/tools/ci/unreferenced_tests_check.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A standalone tests/*.sh that nothing references never runs, and nothing says so. + +Why this exists (engine#1820, behind #1800's withdrawn claim): three PRs in one +night added or touched a shell test that nothing ran -- #1806's +`base_modes_test.sh` (fixed by that PR), #1777's +`release_check_hrx_init_reason_test.sh` (reached main unregistered) and #1809's +unbuilt `add_test`. `ci_targets_check.py` owns "a test is declared and never +runs" but reasons about **targets**, and a standalone `.sh` has none, so it is +blind to the shell half -- which was the majority. + +The rule is one line: every `tests/*.sh` must be named by something that could +run it (the build files, the workflows, `scripts/`, `tools/`, or another test). +A script named only by **itself** does not count: that is the false negative the +first hand-run missed, because the test's own `usage:` line names it. + +Deliberate exclusions live in `tools/ci/unreferenced_tests.txt`, one path and +one reason per line, and are checked in the **other** direction too: an entry +whose file is now referenced, or was deleted, fails rather than rotting there. + + tools/ci/unreferenced_tests_check.py --check [--root .] + tools/ci/unreferenced_tests_check.py --self-test +""" +import argparse +import glob +import os +import sys + +ALLOW = os.path.join("tools", "ci", "unreferenced_tests.txt") +SELF = os.path.join("tools", "ci", "unreferenced_tests_check.py") # this file names tests in its docstring by design + +# What counts as "something that could run it". CMakeLists covers the add_test rows, cmake/ the +# split test files, .github/ the workflows, and scripts/tools/tests everything else that shells out. +SEARCH_TREES = ["cmake", os.path.join(".github", "workflows"), "scripts", "tools", "tests"] +SEARCH_FILES = ["CMakeLists.txt"] + + +def load_allow(root): + """path -> reason, from the allow-list; blank lines and comments are ignored.""" + path = os.path.join(root, ALLOW) + out = {} + if not os.path.exists(path): + return out + for line in open(path, encoding="utf-8"): + line = line.strip() + if not line or line.startswith("#"): + continue + entry, _, reason = line.partition("#") + out[entry.strip()] = reason.strip() + return out + + +def referencing_files(root, need, skips): + """Every file under the search trees that names `need`'s basename. + + `skips` holds the file itself and the allow-list: a test naming its own basename (a `usage:` line) is not + referenced by itself, and the allow-list names every test it excuses, which would otherwise make every + excuse look like a reference and the "stale excuse" arm fire on all of them.""" + base = os.path.basename(need) + skip_norm = {os.path.normpath(os.path.join(root, s)) for s in skips} + # shortcut: a *comment* line naming a test is not a reference, which all these file types spell with `#`. It + # catches the case that made this checker blind to the very test it was written for: its own docstring names + # `release_check_hrx_init_reason_test.sh`, and that counted. Prose in another file's comment can still mask a + # dangling test; upgrade by requiring the name to be on a line that also names a runner, if that ever bites. + hits = set() + files = list(SEARCH_FILES) + for tree in SEARCH_TREES: + for pattern in ("**/*",): + files.extend(glob.glob(os.path.join(root, tree, pattern), recursive=True)) + for rel in files: + full = os.path.join(root, rel) + if not os.path.isfile(full) or os.path.normpath(full) in skip_norm: + continue + try: + text = open(full, encoding="utf-8", errors="ignore").read() + except OSError: + continue + for line in text.splitlines(): + if base in line and not line.lstrip().startswith("#"): + hits.add(rel) + break + return hits + + +def check(root): + tests = sorted(glob.glob(os.path.join(root, "tests", "*.sh"))) + allow = load_allow(root) + unreferenced = [] + for t in tests: + rel = os.path.relpath(t, root) + if not referencing_files(root, t, skips=[t, ALLOW, SELF]): + unreferenced.append(rel) + problems = [] + + # Direction 1: a test nothing names and nobody has explicitly excused. + for rel in unreferenced: + if rel not in allow: + problems.append("%s: referenced by nothing and not in %s -- register it (add_test) or " + "say why not there with the reason" % (rel, ALLOW)) + + # Direction 2: an excuse that no longer excuses anything. + for rel, reason in allow.items(): + if rel not in [os.path.relpath(t, root) for t in tests]: + problems.append("%s: listed in %s but no such test -- delete the entry" % (rel, ALLOW)) + elif rel not in unreferenced: + problems.append("%s: listed in %s but it IS referenced now -- delete the entry" % (rel, ALLOW)) + elif not reason: + problems.append("%s: listed in %s with no reason -- the reason is the point" % (rel, ALLOW)) + + if problems: + print("unreferenced tests (engine#1820):") + for p in problems: + print(" " + p) + return 1 + print("ok %d shell tests, all referenced or excused with a reason (%d excused)" % (len(tests), len(allow))) + return 0 + + +def self_test(): + """A fixture tree: the three outcomes must all be distinguished.""" + import tempfile + import shutil + + def write(root, rel, text): + full = os.path.join(root, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + open(full, "w", encoding="utf-8").write(text) + + fails = 0 + with tempfile.TemporaryDirectory() as d: + write(d, "CMakeLists.txt", 'add_test(NAME good COMMAND tests/good.sh)\n') + write(d, "tests/good.sh", "#!/bin/bash\nexit 0\n") + write(d, "tests/dangling.sh", "#!/bin/bash\n# usage: dangling.sh \nexit 0\n") + write(d, ALLOW, "# comment\ntests/dangling.sh # excused: this fixture says so\n") + if check(d) != 0: + print("FAIL self-test: a dangling test with a reason next to it must pass") + fails += 1 + + # direction 1: drop the excuse -> the dangling test must be reported + write(d, ALLOW, "# nothing excused\n") + if check(d) == 0: + print("FAIL self-test: a dangling test with no excuse must fail") + fails += 1 + else: + print("ok self-test: a dangling test with no excuse is reported") + + # the self-name trap: a file naming only itself must still count as dangling + write(d, ALLOW, "# nothing excused\n") + write(d, "tests/selfnamed.sh", "#!/bin/bash\n# selfnamed.sh runs nothing else\nexit 0\n") + if check(d) == 0: + print("FAIL self-test: a test naming only itself must still be dangling") + fails += 1 + else: + print("ok self-test: a test naming only itself is still dangling") + + # direction 2: an excuse for something referenced, and an excuse for nothing + os.remove(os.path.join(d, "tests/selfnamed.sh")) + write(d, ALLOW, "tests/good.sh # but it is registered\n") + if check(d) == 0: + print("FAIL self-test: an excuse for a registered test must fail") + fails += 1 + else: + print("ok self-test: a stale excuse for a registered test is reported") + write(d, ALLOW, "tests/deleted.sh # gone\n") + if check(d) == 0: + print("FAIL self-test: an excuse for a missing test must fail") + fails += 1 + else: + print("ok self-test: an excuse for a missing test is reported") + + # and the all-clear -- the earlier fixture's dangling test must be gone, or this case is not clean + for leftover in ("dangling.sh", "selfnamed.sh"): + p = os.path.join(d, "tests", leftover) + if os.path.exists(p): + os.remove(p) + write(d, ALLOW, "# nothing excused\n") + write(d, "tests/referenced_elsewhere.sh", "#!/bin/bash\nexit 0\n") + write(d, "scripts/run_it.sh", "#!/bin/bash\nexec tests/referenced_elsewhere.sh\nexit 0\n") + write(d, "tests/dangling2.sh", "#!/bin/bash\nexit 0\n") + write(d, ALLOW, "tests/dangling2.sh # fixture\n") + if check(d) != 0: + print("FAIL self-test: the clean tree must pass") + fails += 1 + else: + print("ok self-test: the clean tree passes") + + print("%s" % ("self-test FAILED" if fails else "self-test ok")) + return 1 if fails else 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--check", action="store_true") + ap.add_argument("--self-test", action="store_true") + ap.add_argument("--root", default=".") + a = ap.parse_args() + if a.self_test: + return self_test() + if a.check: + return check(os.path.abspath(a.root)) + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main())