From 4337c2e34cb78393b90e601641a3f867aaf1d3f7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 04:02:41 -0400 Subject: [PATCH] tests_referenced: the other direction -- a row that names a script which is not in the tree engine#2183 shipped a row whose script a rebase had dropped. Its commit message reads *"the lv3 row script my rebase pattern missed -- the row was Not Run"*, and every gate was green on it: mv tests/whisper_lv3_row.sh tests/whisper_lv3_row.sh.bak tests_referenced_check --check rc 0 tests_referenced_check HERMETIC=1 rc 0 ci_targets_check --check rc 0 `referenced()` is satisfied by the *mention* in CMakeLists.txt, and `ci_targets_check` follows `$` targets rather than plain paths, so a row can name a file that is not there, report **Not Run** for ever, and every gate stays green. That is the same defect as an unreferenced test one step further along: a row that cannot fail. `missing_mentions()` asks the inverse question -- every `tests/*.sh` a RUNNER names must exist -- and joins the same verdict. On the real tree it is **rc 0** (main has no stale mention), and planting one is rc 1: renamed tests/amdxdna_restart_test.sh -> rc 1, one finding naming the row **The false positive that shaped it:** the first regex matched the tail, so `${CMAKE_SOURCE_DIR}/tools/tests/ test-bench-preflight.sh` read as `tests/test-bench-preflight.sh` and reported **two existing files as missing**. `tools/tests/x.sh` is a different file from `tests/x.sh`; the pattern now takes the full path after the variable, and an arm pins exactly that. Three arms added; dropping `missing_mentions` from the verdict fails one. Self-test ok, ctest `tests_referenced` 0.78 s and `tests_referenced_self` 0.04 s, copyright --check rc 0, ci_targets_check --check rc 0. --- tools/ci/tests_referenced_check.py | 52 +++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 4c17c1bf1..e2b373bd8 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -110,6 +110,37 @@ def scan_files(root): continue +# The FULL path after the variable, not the tail that happens to contain `tests/`: `tools/tests/x.sh` is a +# different file from `tests/x.sh`, and clipping the prefix made this report two files that do exist. +MENTION = re.compile(r"\$\{CMAKE_SOURCE_DIR\}/([A-Za-z0-9_./+-]+\.sh)|(?` targets and not plain paths, so **every gate stayed green and + the row reported "Not Run"** for ever. A row that names a file that is not there is a row that cannot fail. + """ + out = [] + for sub in RUNNERS: + base = root / sub + paths = [base] if base.is_file() else sorted(base.rglob("*")) if base.is_dir() else [] + for path in paths: + if path.is_dir() or path.suffix not in SUFFIXES: + continue + try: + text = path.read_text(errors="replace") + except OSError: + continue + for m in MENTION.finditer(text): + mentioned = m.group(1) or m.group(2) + if not (root / mentioned).exists(): + out.append((str(path.relative_to(root)), mentioned)) + return sorted(set(out)) + + def referenced(root, need): """{name: [files]}, ignoring a file mentioning its own basename.""" found = {name: [] for name in need} @@ -248,11 +279,14 @@ def check(root): "CMakeLists.txt or cmake/*.cmake), or add it to %s if it cannot be one yet" % (rel, PYBASE)) for name in py_cleared: print("tests_referenced: %s is in %s and no longer needs to be -- drop the line" % (name, PYBASE)) + for runner, mentioned in missing_mentions(root): + print("tests_referenced: %s names %s, which is not in the tree -- the row exists and can only report " + "Not Run" % (runner, mentioned)) tools_drift = unregistered_selftest_tools(root) for rel in tools_drift: print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in " "CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel) - if drift or gone or idle or py_drift or tools_drift: + if drift or gone or idle or py_drift or tools_drift or missing_mentions(root): return 1 print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) and %d python test(s) are registered" % ( len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))), @@ -300,6 +334,22 @@ def root_with(files): r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"}) cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r + # 1b. the inverse: a row that names a script which is not in the tree. engine#2183 shipped one -- the row + # existed, the mention satisfied the check above, `ci_targets_check` follows $ and not plain + # paths, and the row reported "Not Run" with every gate green. + r = root_with({"tests/ok.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME ok COMMAND bash ${CMAKE_SOURCE_DIR}/tests/ok.sh)\n" + "add_test(NAME gone COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gone.sh)\n"}) + cases.append(("a row naming a script that is not there is drift", check(r), 1, "tests/gone.sh")) + cases_root["a row naming a script that is not there is drift"] = r + + # 1c. and the prefix matters: `tools/tests/x.sh` is a different file from `tests/x.sh`. Clipping the prefix + # here reported two existing files as missing on the real tree, which is the false positive this arm pins. + r = root_with({"tests/ok.sh": "#!/bin/sh\n", "tools/tests/kept.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME ok COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/kept.sh)\n" + "add_test(NAME ok2 COMMAND bash tests/ok.sh)\n"}) + cases.append(("tools/tests/x.sh is not tests/x.sh", check(r), 0, None)) + # 2. referenced from CMakeLists.txt -> clean r = root_with({"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n"}) cases.append(("a registration in CMakeLists.txt counts", check(r), 0, None))