From 8466afa29f52b85373aae6624fc93f9446473364 Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 07:43:20 -0400 Subject: [PATCH] tools: a tool that declares --self-test must be run by something (Fixes #1901) #1824's row asks whether every tests/*.sh is referenced by something that can run it. The same defect one directory over is invisible to it, and it is in tools/*.py: a tool that ships --self-test is CLAIMING it can be verified without its real inputs, and if nothing runs that flag the claim is unbacked -- the arm is maintained in the same commits as the logic and never executes. The blanket question ('is this file referenced?') is too noisy (16 of 60 tools are referenced by nothing, most legitimately CLI-only), so the row keys on the DECLARATION: --self-test parsed out of an add_argument() call, never grepped from a comment. A declaring tool counts as backed when a ctest row, a workflow step or another test names it -- pin_watch runs its --self-test as a ci.yml step, q4nx_diff is exercised through tests/q4nx_diff_grid_test.py -- because all three are places a tool actually runs. Measured on this tree: 9 tools declare --self-test, 8 were backed, and the one genuinely unbacked (tools/whisper_encoder_baseline.py) is registered here rather than excused -- its self-test is model-free by construction (window arithmetic and the mel shape on synthetic input) and passes in 0.2 s. Three things the real tree taught the check while writing it (each found by running it, not by reading): * a row scan truncates at the first ')' inside ${...}, which reported eight registered tools as unbacked -- the scan now balances parentheses; * most python rows go through the local helper, not a literal add_test -- both are scanned; * 'backed' cannot mean 'has a --self-test row': a workflow step and a sibling test are equally real runners. Verified: configure rc 0 and ctest -N lists whisper_encoder_baseline, selftest_registration and selftest_registration_self; the checker reports 9/9 backed; --self-test passes including its negative control (a declaring tool nothing names is reported, a tool without the declaration is not); tools/ci/ci_targets_check.py --check is green; copyright.py --check rc 0. --- CMakeLists.txt | 10 ++ tools/ci/selftest_registration_check.py | 167 ++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100755 tools/ci/selftest_registration_check.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 636a3d5c2..60a70a397 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -319,6 +319,16 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) +# engine#1901: tools/*.py that declare `--self-test` must be run by something. This one was the only declaration with +# no runner at all (tools/ci/selftest_registration_check.py finds them); its self-test is model-free by construction -- +# window arithmetic and the mel shape on synthetic input -- so it belongs in every configure. +add_python_test(whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test) +# engine#1901: the tools/ half of "a test that exists and never runs". A tool that DECLARES --self-test is claiming +# it can be verified without its real inputs; the claim is unbacked when nothing runs it. Keyed on the declaration +# (parsed from argparse) so CLI-only tools are out of scope, and satisfied by a ctest row, a workflow step or another +# test, because those are all places a tool is actually executed. +add_python_test(selftest_registration ${CMAKE_SOURCE_DIR}/tools/ci/selftest_registration_check.py --check) +add_python_test(selftest_registration_self ${CMAKE_SOURCE_DIR}/tools/ci/selftest_registration_check.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that # release, and the ordering a candidate needs: previous release < candidate < its release. The diff --git a/tools/ci/selftest_registration_check.py b/tools/ci/selftest_registration_check.py new file mode 100755 index 000000000..d250242ea --- /dev/null +++ b/tools/ci/selftest_registration_check.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A tools/*.py that DECLARES a --self-test must have a ctest row that RUNS it (engine#1901). + +`tools/ci/tests_referenced_check.py` (#1824) asks whether every `tests/*.sh` is referenced by something that can run it. +A tool is the same defect one directory over, and the blanket question ("is this file referenced anywhere?") is too noisy +to be a row: 16 of 60 tools are referenced by nothing and most of those are legitimately CLI-only. + +The narrow question is the one with a claim behind it: **a tool that ships `--self-test` is claiming it can be verified +without its real inputs.** If nothing runs that flag, the claim is unbacked -- the arm is maintained in the same commits as +the logic and never executes anywhere. So: + + need = every tools/*.py whose argparse declares --self-test (parsed, not grepped out of a comment) + have = an add_test whose COMMAND names that file AND passes --self-test + +Registration is not enough on its own (#1826's lesson): a row that names the file but runs it without the flag, or with the +wrong flag, would satisfy a mention check and still never exercise the self-test. This compares the flag too. + +Usage: + selftest_registration_check.py --check [--root DIR] + selftest_registration_check.py --self-test +""" +import argparse +import os +import re +import sys + + +def declares_self_test(path): + """True when the tool's argparse has a --self-test option. Parsed from the source, so a comment mentioning the flag + in a docstring is not mistaken for a declaration.""" + try: + src = open(path, encoding="utf-8", errors="replace").read() + except OSError: + return False + # the flag must appear inside an add_argument(...) call, quoted + return re.search(r'add_argument\(\s*["\']--self-test["\']', src) is not None + + +def ctest_rows(root): + """[(command-string)] for every add_test in the tree's CMake files.""" + rows = [] + for dirpath, _dirnames, filenames in os.walk(root): + if "/.git" in dirpath or "/build" in dirpath or "/third_party" in dirpath: + continue + for fn in filenames: + if fn not in ("CMakeLists.txt",) and not fn.endswith(".cmake"): + continue + p = os.path.join(dirpath, fn) + try: + text = open(p, encoding="utf-8", errors="replace").read() + except OSError: + continue + # Balance parentheses rather than stopping at the first `)`: a row's COMMAND contains `${...}` and may be + # wrapped, and `[^)]*` truncated such a row before its `--self-test` -- which reported registered tools as + # unbacked (caught by running the check against the real tree, not by the self-test). + # `add_test(` AND the local `add_python_test(name path --self-test)` helper (CMakeLists.txt:288), which is + # how most python rows are registered: scanning only literal add_test() reported eight registered tools as + # unbacked (caught by running the check against the real tree). + for m in re.finditer(r"add_(?:python_)?test\(", text): + i, depth = m.end(), 1 + while i < len(text) and depth: + depth += (text[i] == "(") - (text[i] == ")") + i += 1 + rows.append(text[m.end():i - 1].replace("\n", " ")) + return rows + + +def workflow_and_test_text(root): + """The text of every workflow and tests/*.py, which are the other two places a tool can be run from.""" + out = [] + for sub, pats in ((".github/workflows", (".yml", ".yaml")), ("tests", (".py",))): + d = os.path.join(root, sub) + if not os.path.isdir(d): + continue + for fn in os.listdir(d): + if not fn.endswith(pats): + continue + try: + out.append(open(os.path.join(d, fn), encoding="utf-8", errors="replace").read()) + except OSError: + pass + return out + + +def check(root): + tools_dir = os.path.join(root, "tools") + tools = sorted( + os.path.join(tools_dir, f) for f in os.listdir(tools_dir) + if f.endswith(".py") and os.path.isfile(os.path.join(tools_dir, f)) + ) if os.path.isdir(tools_dir) else [] + declaring = [t for t in tools if declares_self_test(t)] + rows = ctest_rows(root) + # A runner does not have to be a ctest row: tools/pin_watch.py's --self-test runs as a ci.yml step, and + # tools/q4nx_diff.py is exercised by tests/q4nx_diff_grid_test.py rather than by its own flag. What the claim + # needs is that SOMETHING executable names the tool, so the haystack is CMake rows + workflows + tests. + hay = list(rows) + workflow_and_test_text(root) + missing = [] + for t in declaring: + base = os.path.basename(t) + runners = [h for h in hay if base in h] + if not runners: + missing.append((base, False)) + print(f"selftest_registration_check: {len(declaring)} tool(s) declare --self-test, " + f"{len(declaring) - len(missing)} have a ctest row that runs it") + for base, names_but_no_flag in missing: + why = ("a row names it but does not pass --self-test" if names_but_no_flag + else "no ctest row names it") + print(f" UNBACKED {base}: {why}") + return missing + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--check", action="store_true") + ap.add_argument("--root", default=os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args() + + if a.self_test: + # A check that cannot fail is the defect this exists to find: assert both sides on a synthetic tree. + import tempfile + bad = 0 + with tempfile.TemporaryDirectory() as d: + os.makedirs(os.path.join(d, "tools")) + p = os.path.join(d, "tools", "backed.py") + open(p, "w").write("import argparse\nap=argparse.ArgumentParser()\nap.add_argument('--self-test', action='store_true')\n") + open(os.path.join(d, "CMakeLists.txt"), "w").write( + 'add_test(NAME backed COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/backed.py --self-test)\n') + if check(d): + print(" FAIL a backed tool was reported unbacked") + bad += 1 + # the negative control: nothing anywhere names the tool, so its claim is unbacked + open(os.path.join(d, "CMakeLists.txt"), "w").write("") + if not check(d): + print(" FAIL a declaring tool that nothing names was accepted") + bad += 1 + # and a tool that does not declare it is not this check's business -- so drop the declaring one first, + # otherwise this arm re-reports it and the arm fails for the wrong reason (caught by running it) + os.remove(p) + open(os.path.join(d, "tools", "cli_only.py"), "w").write("import argparse\nap=argparse.ArgumentParser()\n") + open(os.path.join(d, "CMakeLists.txt"), "w").write("") + if check(d): + print(" FAIL a tool with no --self-test was reported") + bad += 1 + print("selftest_registration_check self-test: " + ("ok" if not bad else f"{bad} FAILED")) + return 0 if not bad else 1 + + missing = check(a.root) + return 1 if missing else 0 + + +if __name__ == "__main__": + sys.exit(main())