diff --git a/cmake/tests-unreferenced-selftest-rows.cmake b/cmake/tests-unreferenced-selftest-rows.cmake new file mode 100644 index 000000000..cb8fdfcdc --- /dev/null +++ b/cmake/tests-unreferenced-selftest-rows.cmake @@ -0,0 +1,26 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#2456. `tests_referenced_check` was satisfied by a *mention*, so these four tools' `--self-test` arms counted as run while +# nothing ran them: their rows name a test that exercises the tool's logic (`tests/_test.py`), and the tool's own name appeared +# only in the comments beside those rows. Under the narrowed rule they are real debt, and each now has the row it claimed. +# +# The spellings differ and that is the tools', not a typo: `gsm8k_eval.py` and `mtp_depth_pool.py` implement `--selftest` (the +# hyphenated form is an argument error on the first and a *filename* on the second), while `q4nx_diff.py` and `seal_manifest.py` take +# `--self-test`. All four were measured green at these spellings before the rows were written. +add_python_test(gsm8k_eval_self ${CMAKE_SOURCE_DIR}/tools/gsm8k_eval.py --selftest) +add_python_test(mtp_depth_pool_self ${CMAKE_SOURCE_DIR}/tools/mtp_depth_pool.py --selftest) +add_python_test(q4nx_diff_self ${CMAKE_SOURCE_DIR}/tools/q4nx_diff.py --self-test) +add_python_test(seal_manifest_self ${CMAKE_SOURCE_DIR}/tools/seal_manifest.py --self-test) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index e2b373bd8..31a87a317 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -145,8 +145,14 @@ def referenced(root, need): """{name: [files]}, ignoring a file mentioning its own basename.""" found = {name: [] for name in need} for path, text in scan_files(root): + # engine#2456 (cl-flm's arm, PR #2458): the TEST half had the same loophole as the tool half -- a CMake comment naming a + # `tests/*.sh` counted as a runner, so the gate that asks "does anything run this test" was satisfied by prose describing it. + # Comment lines are dropped rather than requiring a full invocation shape: a workflow `run:` line, a Makefile line and a + # ctest row all count legitimately, and only the comment form is the defect. A markdown heading also starts with `#` and loses + # nothing: a script named in a doc's body text is still found, which is what `unregistered_py_tests` relies on. + haystack = "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#")) for name in need: - if name not in text: + if name not in haystack: continue if path.name == name: continue # a test naming itself is not a runner @@ -163,6 +169,70 @@ def referenced(root, need): TOOL_RUNNERS = RUNNERS + ("tests",) +# engine#2456: a mention is not an invocation. The old test was `tool.name in text` over the runner files, so a *comment* that +# named the script satisfied the gate that exists to catch a self-test nobody runs -- measured by writing exactly such a comment: +# green with the comment, red once it was reworded, green again with a real row. +INVOCATION = re.compile(r"add_test|add_python_test|\bCOMMAND\b|\bpython3?\b|--self-test|\bbash\b") + + +def _strip_line_comments(text): + """Drop `#...` to end of line before any depth walk, outside a quoted string. + + The review of engine#2460 measured the need on the engine's own file: `CMakeLists.txt` has a comment containing an unclosed `(` + (line 2562, "# \"By name\" means a line CMake would execute -- include( at the start of a line …"), so the depth walk that counted + parentheses inside comments swallowed the remaining 51 lines into one statement. That statement starts with `#`, so `invokes()` + skipped it as a comment and **the tail of the file became invisible to the gate** -- exactly where rows get appended. Nothing + registered lived there yet, which is why it read green; the next row added would have been reported as unregistered while registered. + """ + out = [] + for line in text.splitlines(): + quoted = False + cut = len(line) + for i, ch in enumerate(line): + if ch == '"': + quoted = not quoted + elif ch == "#" and not quoted: + cut = i + break + out.append(line[:cut]) + return "\n".join(out) + + +def _statements(text): + """Logical statements: backslash continuations joined, and newlines inside parentheses collapsed. + + The parenthesised collapse is the part a one-line scan gets wrong (engine#2455): CMake wraps `add_test(NAME x` and + `COMMAND .../tool.py` onto separate lines with no backslash, so a line-at-a-time match reports four genuinely-registered + scripts as unregistered. + """ + text = _strip_line_comments(text) + text = re.sub(r"\\\n", " ", text) + out, depth = [], 0 + for ch in text: + if ch == "(": + depth += 1 + elif ch == ")" and depth: + depth -= 1 + if ch == "\n" and depth: + out.append(" ") + continue + out.append(ch) + return "".join(out).splitlines() + + +def invokes(name, text): + """True when a statement names the script AND carries an invocation shape; comment lines are mentions, not invocations.""" + if name in text and not INVOCATION.search(text): + return False # cheap out: a file with no invocation vocabulary cannot invoke anything + for line in _statements(text): + stripped = line.strip() + if stripped.startswith(("#", "//", "*", "