From dd2f2b803dbd82704f794b579606046c2301dd57 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 19:57:16 -0400 Subject: [PATCH] tests_referenced: an invocation, not a mention -- a comment naming a script's --self-test no longer satisfies it (engine#2456) `tests_referenced_check` asks, for every `tools/**/*.py` and `scripts/*.sh` that declares a self-test, whether any file under a runner directory **contains the script's name**. `tests/` is one of those directories, so a *comment* that named the script satisfied it -- i.e. the class the check exists to prevent ("a self-test nobody runs", engine#2429/#2418/#2416) could be re-introduced by documenting the script. Found by writing one, on engine#2455: ``` comment in tests/pack_modes_test.sh naming scripts/window-run.sh -> green (wrong: no row existed) comment reworded so it does not name the file -> red (right) add_test(NAME window_run_self ...) -> green; delete the row -> red again ``` The rule now requires the script's name **and** a self-test flag on the **same non-comment line**. Two details that took a correction each: - **both spellings**, `--self-test` and `--selftest`, because the tree uses both; the invocation side now reuses the same `SELFTEST` regex the declaration side uses, so a tool declaring one form cannot read as unreferenced forever; - **comment lines excluded** (`#` and `//` after leading whitespace). Requiring the flag alone was not enough: a comment that names the script *with* the flag is exactly the shape that read green for months. **What the stricter rule found on main** -- the four tools whose self-test no row ever ran, matching the calibration in the issue: `tools/ci/dupdef_check.py`, `tools/gsm8k_eval.py`, `tools/q4nx_diff.py`, `tools/seal_manifest.py`. Rows added for all four (each with its own flag spelling; `gsm8k_eval` is the one that spells it `--selftest`), because a check that gets stricter has to leave main green under the stricter rule. ``` $ python3 tools/ci/tests_referenced_check.py --check tests_referenced: all 91 test script(s) are referenced (1 deliberately not a row: anchor_rerun.sh); 56 tool self-test(s) ... $ python3 tools/ci/tests_referenced_check.py --self-test tests_referenced self-test: 0 failure(s) ``` **Two arms, and the pair is the specification**: the same tree with the name only in a comment is **drift**, and with a real `add_test` row it is **clean**. Verified by mutation, not by inspection -- dropping `and not is_comment(ln)` turns the first arm red: | tree | self-test | |---|---| | this commit | 0 failure(s) | | comment exclusion removed | **1 failure** (`a COMMENT naming a tool --self-test is still drift`) | That mutant is also why the first version of this fix was wrong: requiring the flag but not excluding comments passed the negative case only because my fixture's comment happened to omit the flag. --- CMakeLists.txt | 9 ++++++++ tools/ci/tests_referenced_check.py | 37 +++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fb5eae707..4c1b1a06b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -289,6 +289,15 @@ add_test(NAME kernel_set_check COMMAND bash ${CMAKE_SOURCE_DIR}/tests/kernel_set # gets one row. add_test(NAME dupdef_check COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/dupdef_check.py --check WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +# engine#2456: `tests_referenced_check` used to accept a MENTION of a script as an invocation, so four tools whose +# `--self-test`/`--selftest` no row ever ran read as referenced -- by a comment in a test file. It now requires the +# script's name and a self-test flag on the same line, which is what these four rows are: the check got stricter and +# main has to be green under the stricter rule. Each flag spelling is the tool's own (`gsm8k_eval` spells it without +# the hyphen; the other three with it). +add_test(NAME dupdef_check_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/dupdef_check.py --self-test) +add_test(NAME gsm8k_eval_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/gsm8k_eval.py --selftest) +add_test(NAME q4nx_diff_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/q4nx_diff.py --self-test) +add_test(NAME seal_manifest_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/seal_manifest.py --self-test) # engine#2255: `--help` must not be a traceback. Five tools indexed sys.argv directly, so the flag # arrived as a filename -- one with IndexError, the others with `FileNotFoundError: '--help'`, and two # of those on `--help/dd_metastate_...`, which is what a path join with the flag looks like. Fixed; this diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index e2b373bd8..5fdd03fab 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -155,6 +155,13 @@ def referenced(root, need): SELFTEST = re.compile(r"--self[-_]?test\b") + +# engine#2456: a runner line that only *documents* a script is not a runner. Both comment styles appear in the tree +# (`#` in CMake/shell/YAML, `//` in C++); a line starting with either, after leading whitespace, cannot invoke +# anything, so a match there is a mention. +def is_comment(line): + t = line.lstrip() + return t.startswith("#") or t.startswith("//") RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows") # A tool's `--self-test` can also be run from a python test, so THAT gate reads one more directory. The python-test gate must # not: with "tests" in the shared tuple, `tests/lonely_test.py` mentioning its own name satisfied "some runner names it" and the @@ -179,6 +186,12 @@ def unregistered_selftest_tools(root): cand += [p for p in scripts.glob("*.sh")] if not cand: return [] + # engine#2456: a MENTION is not an INVOCATION. `tests/` is a runner directory, so a comment in a test file that + # names the script used to satisfy this check -- i.e. the class it exists to prevent ("a self-test nobody runs", + # engine#2429/#2418/#2416) could be re-introduced by documenting the script. Reproduced on #2455: a comment in + # tests/pack_modes_test.sh naming scripts/window-run.sh read green while no row existed, and rewording the + # comment read red -- the check was answering "does this string appear" where it meant "does a runner run it". + # So collect LINES, not blobs, and count only a line that names the script AND passes --self-test. runner_text = [] for sub in TOOL_RUNNERS: p = root / sub @@ -188,6 +201,7 @@ def unregistered_selftest_tools(root): runner_text.append(f.read_text(errors="replace")) except OSError: pass + runner_lines = [ln for text in runner_text for ln in text.splitlines()] drift = [] for tool in sorted(cand): try: @@ -196,7 +210,12 @@ def unregistered_selftest_tools(root): continue if not SELFTEST.search(src): continue - if not any(tool.name in text for text in runner_text): + # Both spellings occur in the tree (`--self-test` and `--selftest`); the same regex the declaration side + # uses must decide the invocation side, or a tool declaring one form reads as unreferenced forever. + # AND the line must not be a comment: a comment naming the script WITH the flag is precisely the case that + # read green for months (`# see tools/x.py --self-test for the arms`), and it is the shape the self-test's + # first arm pins -- requiring the flag alone did not catch it, excluding comment lines does. + if not any(tool.name in ln and SELFTEST.search(ln) and not is_comment(ln) for ln in runner_lines): drift.append(str(tool.relative_to(root))) return drift @@ -364,6 +383,22 @@ def root_with(files): ".github/workflows/ci.yml": "run: bash tests/ci.sh\n"}) cases.append(("a mention in a workflow counts", check(r), 0, None)) + # 4a. engine#2456: a MENTION is not an INVOCATION. A comment in a runner file that names the script used to + # satisfy this check -- reproduced on #2455, where a comment in tests/pack_modes_test.sh naming + # scripts/window-run.sh read green while no row existed. The pair below is the whole rule: the same tree, once + # with the name only in a comment (drift) and once with a real row (clean), so a regression to substring + # matching fails the first arm while anything that only checks the positive case still passes. + tool = "import sys\nif '--self-test' in sys.argv: pass\n" + row_t = "add_test(NAME t COMMAND bash tests/t.sh)\n" + r = root_with({"tools/mine.py": tool, + "tests/t.sh": "#!/bin/sh\n# see tools/mine.py --self-test for the arms\n", + "CMakeLists.txt": row_t}) + cases.append(("a COMMENT naming a tool --self-test is still drift", check(r), 1, "tools/mine.py")) + cases_root["a COMMENT naming a tool --self-test is still drift"] = r + r = root_with({"tools/mine.py": tool, "tests/t.sh": "#!/bin/sh\n", + "CMakeLists.txt": row_t + "add_test(NAME mine COMMAND python3 tools/mine.py --self-test)\n"}) + cases.append(("...and a real add_test row clears the same tree", check(r), 0, None)) + # 4b. a PYTHON test naming only itself is drift even though `tests/` is a run site for TOOL self-tests -- the two gates # read different corpora on purpose (measured: sharing one tuple let tests/lonely_test.py register itself) r = root_with({"tests/lonely_test.py": "import sys\n# usage: tests/lonely_test.py\nprint('--self-test')\n",