Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,13 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test.
# two files, no build and no submodules; --self-test drives it.
add_test(NAME ci_targets COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR})
add_test(NAME ci_targets_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test)
# engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail.
# `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and
# tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink.
add_test(NAME tests_referenced COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --check --root ${CMAKE_SOURCE_DIR})
add_test(NAME tests_referenced_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --self-test)
# engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820).
add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh)
# engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three
# orphans of that shape landed in two days (51 GiB of GTT for 14 h; an NPU server 70 min past a 6-min ticket;
# then two lanes each starting an HRX server outside the token within one half hour). The ancestry test is the
Expand Down
208 changes: 208 additions & 0 deletions tools/ci/tests_referenced_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Every tests/*.sh is mentioned by something that can run it (engine#1820).

`ci_targets_check.py` asks this question about *targets* -- a test whose executable CI never builds. It cannot
see the other half: a standalone shell test that nothing references at all. Three of those turned up on 2026-11-10
in one night (#1806 registered one that had no runner; #1777 added one nothing referenced; #1809 added one whose
target ci.yml never builds, which is the case ci_targets_check does cover). A test nobody runs cannot fail, so it
reads as green for ever -- the same defect as #1595, one file kind over.

need = every tests/*.sh
have = every file that MENTIONS its basename, in a place that can run things:
CMakeLists.txt, cmake/**, scripts/**, tools/**, tests/**, .github/workflows/**
drift = need - have - allow-list

A mention inside the file itself does not count: a test's own header or usage line naming itself is not a runner,
and counting it would make the whole check vacuous (measured on this tree: it is the one mistake the hand-written
loop made first).

Documentation is deliberately NOT a reference: a sentence in a `.md` that names a test is not something that runs it.

The allow-list is `tools/ci/unreferenced_tests.txt`, one basename per line with an optional `# reason` -- the same
shape as `add_test_gaps.txt`. An entry that no longer needs to be there is reported so the list can shrink, which
keeps it honest rather than accumulating.

usage: tools/ci/tests_referenced_check.py [--check] [--self-test] [--root DIR]
Exit: 0 every test is reachable, 1 an unreferenced one (named), 2 cannot tell (no tests/ directory).
"""

import argparse
import pathlib
import sys

ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt")
SCAN = ("", "cmake", "scripts", "tools", "tests", ".github/workflows")
SUFFIXES = {".txt", ".cmake", ".yml", ".yaml", ".sh", ".py"}


def allow_list(root):
"""{basename: reason} from the allow-list; comments and blanks skipped."""
path = root / ALLOW
out = {}
if not path.is_file():
return out
for line in path.read_text(errors="replace").splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
name, _, reason = line.partition("#")
out[name.strip()] = reason.strip()
return out


def scan_files(root):
"""Every file that can reference a test, as (path, text) -- docs and the allow-list excluded."""
for sub in SCAN:
base = root / sub if sub else root
if not base.is_dir():
continue
pattern = "*" if sub else "CMakeLists.txt"
for path in sorted(base.glob(pattern)):
if path.is_dir() or path.suffix not in SUFFIXES:
continue
if path == root / ALLOW:
continue # the allow-list names the file too; it is not a runner
try:
yield path, path.read_text(errors="replace")
except OSError:
continue


def referenced(root, need):
"""{name: [files]}, ignoring a file mentioning its own basename."""
found = {name: [] for name in need}
for path, text in scan_files(root):
for name in need:
if name not in text:
continue
if path.name == name:
continue # a test naming itself is not a runner
found[name].append(str(path.relative_to(root)))
return found


def check(root):
tests = root / "tests"
if not tests.is_dir():
print("tests_referenced: cannot tell: no tests/ directory under %s" % root, file=sys.stderr)
return 2
need = sorted(p.name for p in tests.glob("*.sh"))
allow = allow_list(root)
found = referenced(root, need)
drift = [n for n in need if not found[n] and n not in allow]
stale = [n for n in allow if n not in need]
for name in drift:
print("tests_referenced: %s is referenced by nothing that can run it -- register it (add_test in "
"CMakeLists.txt or cmake/*.cmake), or list it in %s with the reason it must not be a row" % (name, ALLOW))
for name in stale:
print("tests_referenced: %s is in %s but no such test exists any more -- drop the line" % (name, ALLOW))
if drift:
return 1
print("tests_referenced: all %d test script(s) are referenced%s" % (
len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow)))))
return 0


def self_test():
"""Synthetic roots: a check that only ever sees a good tree proves nothing."""
import tempfile

cases = []

def root_with(files):
d = pathlib.Path(tempfile.mkdtemp())
for rel, body in files.items():
p = d / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(body)
return d

# 1. a test nobody mentions -> drift, and it is NAMED
r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"})
cases_root = {}
cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r

# 2. referenced from CMakeLists.txt -> clean
r = root_with({"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n"})
cases.append(("a registration in CMakeLists.txt counts", check(r), 0, None))

# 3. referenced from an included family file -> clean
r = root_with({"tests/fam.sh": "#!/bin/sh\n", "CMakeLists.txt": "",
"cmake/tests-family.cmake": "add_test(NAME fam COMMAND bash tests/fam.sh)\n"})
cases.append(("a registration in cmake/*.cmake counts", check(r), 0, None))

# 4. referenced from a workflow -> clean
r = root_with({"tests/ci.sh": "#!/bin/sh\n", "CMakeLists.txt": "",
".github/workflows/ci.yml": "run: bash tests/ci.sh\n"})
cases.append(("a mention in a workflow counts", check(r), 0, None))

# 5. a test naming ONLY itself -> drift (the vacuous case)
r = root_with({"tests/self.sh": "#!/bin/sh\n# usage: tests/self.sh\n", "CMakeLists.txt": ""})
cases.append(("a test naming itself is not a runner", check(r), 1, "self.sh")); cases_root["a test naming itself is not a runner"] = r

# 6. allow-listed with a reason -> clean, and the reason is carried
r = root_with({"tests/keeper.sh": "#!/bin/sh\n", "CMakeLists.txt": "",
str(ALLOW): "keeper.sh # an operator measurement: always exits 0, so a row would be green noise\n"})
cases.append(("an allow-listed test is not drift", check(r), 0, None))

# 7. a stale allow entry -> reported, but not a failure
r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "tests/one.sh\n",
str(ALLOW): "gone.sh # removed long ago\n"})
cases.append(("a stale allow entry is progress, not drift", check(r), 0, "gone.sh")); cases_root["a stale allow entry is progress, not drift"] = r

# 8. no tests/ at all -> cannot tell, not "clean"
r = root_with({"CMakeLists.txt": ""})
cases.append(("no tests/ directory is 'cannot tell'", check(r), 2, None))

failures = 0
import contextlib
import io
for what, got, want, mentions in cases:
if got != want:
print("FAIL %s: rc %s, wanted %s" % (what, got, want))
failures += 1
continue
if mentions:
# re-run capturing the output: the verdict must NAME the offending test, or a reader cannot act on it
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
if what.startswith("a stale"):
check(cases_root[what])
else:
check(cases_root[what])
if mentions not in buf.getvalue():
print("FAIL %s: rc was right but the output does not name %s" % (what, mentions))
failures += 1
continue
print("ok %s" % what)
print("tests_referenced self-test: %d failure(s)" % failures)
return 1 if failures else 0


def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--check", action="store_true")
ap.add_argument("--self-test", action="store_true")
ap.add_argument("--root", default=".", help="tree to inspect (default: the current directory)")
a = ap.parse_args()
if a.self_test:
return self_test()
return check(pathlib.Path(a.root).resolve())


if __name__ == "__main__":
sys.exit(main())
21 changes: 21 additions & 0 deletions tools/ci/unreferenced_tests.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# tests/*.sh that are deliberately NOT a ctest row, one basename per line, with the reason it must stay out.
# `tools/ci/tests_referenced_check.py` fails when a test is referenced by nothing that can run it, and reports a
# line here that is no longer needed so this list can shrink. The bar for an entry is the same one #1806 used:
# a ctest row has to be ABLE TO FAIL. A script that always exits 0 is an operator measurement, not a test.
#
anchor_rerun.sh # an operator measurement: needs real GGUFs and an output dir, allows 30 min per model, prints PASS/FAIL/SKIP per model and ALWAYS exits 0 -- as a ctest row it would either skip for ever or report green over a FAIL line (engine#1800)
3 changes: 3 additions & 0 deletions tools/copyright.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
# engine#1595: the direction-B ratchet's baseline is a '#'-comment data file too, and `.txt` has no style of
# its own (a blanket "*.txt" would demand a notice in files that are data, not code).
"add_test_gaps.txt",
# engine#1820: the other sweep's allow-list, same shape and the same reason -- `#`-comment data, `.txt` has no
# style, and the header it wants is already at the top of the file.
"unreferenced_tests.txt",
"21serial-console", "90onboard-limits", "Dockerfile", "*.Dockerfile",
# cycle 17: Puppy's switch that skips its first-run Quick Setup is an empty marker file whose only content
# is this notice - Puppy tests for its existence, so a comment is free
Expand Down
Loading