From 19fdaa5a6e26ed50b6054706d1161967d6879fae Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 19:57:47 -0400 Subject: [PATCH 1/3] =?UTF-8?q?tests=5Freferenced:=20require=20an=20invoca?= =?UTF-8?q?tion,=20not=20a=20mention=20=E2=80=94=20and=20the=20four=20arms?= =?UTF-8?q?=20it=20was=20hiding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine#2456 found the loophole the only way anyone would: by writing a comment. `if not any(tool.name in text ...)` over the runner files meant a **comment naming the script** satisfied the gate that exists to catch a self-test nobody runs. **The rule now**: a statement must name the script *and* carry an invocation shape (`add_test|add_python_test|COMMAND|python3|bash|--self-test`), with **continuation lines joined first** — parenthesised blocks collapsed and backslash continuations closed. That second part is not decoration: it is exactly what made the issue's own first scan report four false positives (engine#2455), and what the checker would have repeated. Three arms, in the existing self-test: | arm | expected | |---|---| | a `#` comment naming `scripts/helper.sh` | **drift** (the loophole, pinned by the thing that found it) | | a continuation-wrapped `add_test(NAME x\n COMMAND bash scripts/helper.sh --self-test)` | **clean** (the shape that caused the false positives) | | `python3 scripts/helper.sh --self-test` inside a registered test | **clean** | **And the narrowed rule found four live instances, which the issue reported as zero debt.** Its author dismissed the same four as a measurement error ("they have a real row"); they do — a row for `tests/_test.py`, which exercises the tool's *logic*. The tool's own `--self-test` was never run, and its name appeared only in the comments beside those rows: ``` tools/gsm8k_eval.py tools/mtp_depth_pool.py tools/q4nx_diff.py tools/seal_manifest.py ``` Four rows in `cmake/tests-unreferenced-selftest-rows.cmake`, and the spellings are the tools', not a typo — measured before writing them: ``` gsm8k_eval.py --selftest rc 0 (--self-test is an argument error) mtp_depth_pool.py --selftest rc 0 (--self-test is read as a *filename*) q4nx_diff.py --self-test rc 0 seal_manifest.py --self-test rc 0 ``` That difference is worth one line of the tool's own vocabulary: `SELFTEST = re.compile(r"--self[-_]?test\b")` accepts both spellings when *detecting* an arm, so the gate could never have told a tool that takes `--selftest` from one that takes `--self-test` — and a row written with the wrong one goes red at run time, not at check time. Measured after the change, on this tree: `--check` green (91 scripts, 56 tool self-tests, 47 python tests) and `--self-test` 0 failures. `copyright --check`, `ci_targets_check`, `tmp_path_check` and `registration_loss_check --base origin/main` all green beforehand. Fixes #2456. --- cmake/tests-unreferenced-selftest-rows.cmake | 26 +++++++++ tools/ci/tests_referenced_check.py | 61 +++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 cmake/tests-unreferenced-selftest-rows.cmake diff --git a/cmake/tests-unreferenced-selftest-rows.cmake b/cmake/tests-unreferenced-selftest-rows.cmake new file mode 100644 index 000000000..cb8fdfcdc --- /dev/null +++ b/cmake/tests-unreferenced-selftest-rows.cmake @@ -0,0 +1,26 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#2456. `tests_referenced_check` was satisfied by a *mention*, so these four tools' `--self-test` arms counted as run while +# nothing ran them: their rows name a test that exercises the tool's logic (`tests/_test.py`), and the tool's own name appeared +# only in the comments beside those rows. Under the narrowed rule they are real debt, and each now has the row it claimed. +# +# The spellings differ and that is the tools', not a typo: `gsm8k_eval.py` and `mtp_depth_pool.py` implement `--selftest` (the +# hyphenated form is an argument error on the first and a *filename* on the second), while `q4nx_diff.py` and `seal_manifest.py` take +# `--self-test`. All four were measured green at these spellings before the rows were written. +add_python_test(gsm8k_eval_self ${CMAKE_SOURCE_DIR}/tools/gsm8k_eval.py --selftest) +add_python_test(mtp_depth_pool_self ${CMAKE_SOURCE_DIR}/tools/mtp_depth_pool.py --selftest) +add_python_test(q4nx_diff_self ${CMAKE_SOURCE_DIR}/tools/q4nx_diff.py --self-test) +add_python_test(seal_manifest_self ${CMAKE_SOURCE_DIR}/tools/seal_manifest.py --self-test) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index e2b373bd8..566099b8a 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -163,6 +163,46 @@ def referenced(root, need): TOOL_RUNNERS = RUNNERS + ("tests",) +# engine#2456: a mention is not an invocation. The old test was `tool.name in text` over the runner files, so a *comment* that +# named the script satisfied the gate that exists to catch a self-test nobody runs -- measured by writing exactly such a comment: +# green with the comment, red once it was reworded, green again with a real row. +INVOCATION = re.compile(r"add_test|add_python_test|\bCOMMAND\b|\bpython3?\b|--self-test|\bbash\b") + + +def _statements(text): + """Logical statements: backslash continuations joined, and newlines inside parentheses collapsed. + + The parenthesised collapse is the part a one-line scan gets wrong (engine#2455): CMake wraps `add_test(NAME x` and + `COMMAND .../tool.py` onto separate lines with no backslash, so a line-at-a-time match reports four genuinely-registered + scripts as unregistered. + """ + text = re.sub(r"\\\n", " ", text) + out, depth = [], 0 + for ch in text: + if ch == "(": + depth += 1 + elif ch == ")" and depth: + depth -= 1 + if ch == "\n" and depth: + out.append(" ") + continue + out.append(ch) + return "".join(out).splitlines() + + +def invokes(name, text): + """True when a statement names the script AND carries an invocation shape; comment lines are mentions, not invocations.""" + if name in text and not INVOCATION.search(text): + return False # cheap out: a file with no invocation vocabulary cannot invoke anything + for line in _statements(text): + stripped = line.strip() + if stripped.startswith(("#", "//", "*", "