From 239d04ae55006de26fec5761f7dc2fd2474cf385 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 07:21:43 -0400 Subject: [PATCH 1/2] tests_referenced: a tests/*.py that invokes a tool's --self-test is a run site (port of #1912's one extra dimension) tools/ci/tests_referenced_check.py's RUNNERS are the places a self-test can be run from: the root CMakeLists, cmake/, and the workflows. A python test is a fourth, and it is missing. Measured on a fixture where tools/lonely.py's --self-test is invoked only from tests/invokes_test.py (itself registered in cmake/tests-x.cmake): without this change tools/lonely.py declares a self-test and no runner names it -- ... the arm is dead code with it all 0 test script(s) are referenced; 1 tool self-test(s) and 1 python test(s) are registered So it is a false positive, and a misleading one: the fix a reader would apply is a redundant ctest row for an arm that already runs, which is how a corpus grows rows nobody needs. Four characters of behaviour -- "tests" added to RUNNERS -- because the haystack is already built by walking RUNNERS generically. engine#1901's tools half and engine#2129's scripts half both stay; this is the run-site half. Held rather than opened: engine PR #1912 adds a whole second checker (167 lines, tools-only) for the same property, so the order should be this port first, then #1912 closed as superseded -- its one genuine extra dimension is what is ported here. --- tools/ci/tests_referenced_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index e0ed420ff..ce013b880 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -124,7 +124,7 @@ def referenced(root, need): SELFTEST = re.compile(r"--self[-_]?test\b") -RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows") +RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows", "tests") def unregistered_selftest_tools(root): From 1d81a25226b30dae35525ebafba01edf15a6a529 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 17:31:24 -0400 Subject: [PATCH 2/2] tests_referenced: the tests/ run site belongs to the tool-self-test gate, not the shared corpus Caught in my own change before it merged. Adding "tests" to `RUNNERS` gave BOTH gates that haystack, and there are two gates: unregistered_selftest_tools() asking "does anything run this tool's --self-test?" unregistered_py_tests() asking "does anything run this python test?" The python-test gate's "registered" test is `any(q.name in t for t in runner_text)`; with `tests/` in the haystack, a test's own file is in the text it is searched against, so `tests/lonely_test.py` mentioning its own path **registered itself**: before the split: tests_referenced: ... 1 python test(s) are registered <- the self-naming test passed after the split: tests_referenced: tests/lonely_test.py is a test that no runner names -- add a ctest row ... The guard against a test naming itself lives in the path-shaped bookkeeping of the other function, not in this haystack, so sharing one tuple removed a protection without touching the line that documents it. The corpora are now separate: RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows") TOOL_RUNNERS = RUNNERS + ("tests",) <- the tool-self-test gate only Both directions measured on fixtures: a tool whose --self-test is invoked ONLY from a registered python test -> registered (the improvement, kept) a python test that names only itself -> drift (the regression, fixed, arm added) The arm fails if the two corpora are ever merged again (`RUNNERS = TOOL_RUNNERS` -> 1 FAILURE), so this cannot come back silently. `--self-test` 0 failures; the real tree still reports 86 test scripts, 44 tool self-tests and 45 python tests. --- tools/ci/tests_referenced_check.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index ce013b880..4c17c1bf1 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -124,7 +124,12 @@ def referenced(root, need): SELFTEST = re.compile(r"--self[-_]?test\b") -RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows", "tests") +RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows") +# A tool's `--self-test` can also be run from a python test, so THAT gate reads one more directory. The python-test gate must +# not: with "tests" in the shared tuple, `tests/lonely_test.py` mentioning its own name satisfied "some runner names it" and the +# test registered itself (measured on a fixture -- the guard against a test naming itself lives in the other function's path +# shape, not in this haystack). engine#1901's tools half and engine#2129's scripts half are unchanged. +TOOL_RUNNERS = RUNNERS + ("tests",) def unregistered_selftest_tools(root): @@ -144,7 +149,7 @@ def unregistered_selftest_tools(root): if not cand: return [] runner_text = [] - for sub in RUNNERS: + for sub in TOOL_RUNNERS: p = root / sub files = [p] if p.is_file() else (sorted(q for q in p.rglob("*") if q.is_file()) if p.is_dir() else []) for f in files: @@ -309,6 +314,13 @@ def root_with(files): ".github/workflows/ci.yml": "run: bash tests/ci.sh\n"}) cases.append(("a mention in a workflow counts", check(r), 0, None)) + # 4b. a PYTHON test naming only itself is drift even though `tests/` is a run site for TOOL self-tests -- the two gates + # read different corpora on purpose (measured: sharing one tuple let tests/lonely_test.py register itself) + r = root_with({"tests/lonely_test.py": "import sys\n# usage: tests/lonely_test.py\nprint('--self-test')\n", + "CMakeLists.txt": ""}) + cases.append(("a python test that names only itself is still drift", check(r), 1, "lonely_test.py")) + cases_root["a python test that names only itself is still drift"] = r + # 5. a test naming ONLY itself -> drift (the vacuous case) r = root_with({"tests/self.sh": "#!/bin/sh\n# usage: tests/self.sh\n", "CMakeLists.txt": ""}) cases.append(("a test naming itself is not a runner", check(r), 1, "self.sh")); cases_root["a test naming itself is not a runner"] = r