From 475ba3ef6bbd654fbde648107f42c7f63dd34944 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 07:42:51 -0400 Subject: [PATCH] tests_referenced: a tools/*.py that declares a self-test and is registered nowhere is now drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine#1901. `#1824`'s sweep asks whether every `tests/*.sh` is referenced by something that can run it, and it is specific to `tests/*.sh` -- so the same defect one directory over was invisible. A tool that ships `--self-test` is claiming it can be verified without its real inputs; if nothing runs that flag the claim is unbacked, and the arm is maintained in the same commits as the logic while never executing. **Narrow on purpose.** A blanket "every `tools/*.py` is referenced" sweep is 16 of 60 today, most of them legitimately CLI-only, which is too noisy to be a ratchet. The rule is "declares a self-test flag AND no registration names it", and it found **five**, not the two the issue measured: batched_repeat --selftest logprob_agree --selftest mtp_head_ref --selftest template_bom --self-test whisper_encoder_baseline --self-test All five self-tests pass on this box, so all five are now `add_python_test` rows (the helper #1891 added), and the checker fails if any future one loses its registration. **The flag spelling is per tool, and that is the point.** Three take `--selftest`, two `--self-test`; a registration with the wrong one exits non-zero and fails loudly in ctest rather than silently running nothing. That split is the same one #1886's review flagged and #1901 describes, so the rows carry a comment naming which spelling each tool accepts. **Two false positives found and closed while writing the rule** -- both would have made it noise on the first run: * `tools/spec_decode_acceptance.py` **is** run: `CMakeLists.txt:857` registers `tests/spec_decode_acceptance_test.py`, which imports the tool. A registration that names the *test* counts, so `tests/*.py` is part of the registration set. * and an import names the **stem** (`import spec_decode_acceptance`), not the file, so the match accepts either the basename or the stem. Verified: `tests_referenced_check.py --check` was **rc 1** with five names before, **rc 0** after ("all 73 test script(s) are referenced, 1 deliberately not a row: anchor_rerun.sh"); `--self-test` **0 failures**, with three new arms (an unregistered self-test is drift and is NAMED, a registered one is clean, and a tool a registered `tests/*.py` imports is run -- the last two exist so the first cannot pass by "tools/ exists"); each of the five registered commands run by hand → rc 0; and `ci_targets_check --check` rc 0, since these rows name no build target. --- CMakeLists.txt | 10 ++++ tools/ci/tests_referenced_check.py | 76 +++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 636a3d5c2..305bf09e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -725,6 +725,16 @@ add_python_test(copyright_check_untracked ${CMAKE_SOURCE_DIR}/tests/copyright_ch # loaded box yields VOID instead of a number. The self-test is the estimator's own gate and needs no GPU: it accepts # agreement, refuses disagreement, and proves a dropped run cannot shift an arm. add_python_test(fa_ratio_estimator ${CMAKE_SOURCE_DIR}/tools/fa_ratio.py --self-test) +# engine#1901: five tools that DECLARE a self-test and were registered nowhere -- the arm existed, was maintained with the +# logic, and never executed. `tools/ci/tests_referenced_check.py` now fails on that state (a tools/*.py with `--self-test` +# that no registration names), so this list cannot rot back. Note the flag spelling is per tool, not a convention: three +# take `--selftest` and two `--self-test`, which is the split engine#1901 exists to make visible -- a registration using +# the wrong one exits non-zero and fails loudly here rather than silently running nothing. +add_python_test(batched_repeat ${CMAKE_SOURCE_DIR}/tools/batched_repeat.py --selftest) +add_python_test(logprob_agree ${CMAKE_SOURCE_DIR}/tools/logprob_agree.py --selftest) +add_python_test(mtp_head_ref ${CMAKE_SOURCE_DIR}/tools/mtp_head_ref.py --selftest) +add_python_test(template_bom ${CMAKE_SOURCE_DIR}/tools/template_bom.py --self-test) +add_python_test(whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test) add_dependencies(onebit route_log_test) # CI builds `--target onebit` plus a fixed list in .github/workflows/ci.yml; building this check with onebit # keeps the laya_models ctest from reporting "Not Run" without a workflow change. diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 90ba398f8..5fe3fb020 100755 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -98,6 +98,55 @@ def referenced(root, need): return found +SELF_TEST_FLAGS = ("--self-test", "--selftest") + + +def tools_with_selftest(root): + """tools/*.py that DECLARE a self-test flag (engine#1901). + + Narrow on purpose: a blanket "every tools/*.py is referenced" sweep is 16 of 60 today and most of those are legitimate + CLI-only tools, which is too noisy to be a ratchet. A tool that ships `--self-test` makes a different claim -- that it + can be verified without its real inputs -- and a claim nothing runs is not backed: the arm exists, is maintained with the + logic in the same commits, and never executes. + """ + out = [] + tools = root / "tools" + if not tools.is_dir(): + return out + for path in sorted(tools.glob("*.py")): + try: + text = path.read_text(errors="replace") + except OSError: + continue + if any(flag in text for flag in SELF_TEST_FLAGS): + out.append(path.name) + return out + + +def registered_tools(root, need): + """{name: [files]} naming each tool in a REGISTRATION file -- CMakeLists.txt or cmake/*.cmake. + + Not `referenced()`: a tool mentioned in a script, a doc or another tool is not run by anything. The claim being checked + is that a registration exists, so only the two places that register tests count. + """ + found = {name: [] for name in need} + regs = [root / "CMakeLists.txt"] + sorted((root / "cmake").glob("*.cmake")) + # A tests/*.py counts too: a registered test that imports the tool runs it, and `spec_decode_acceptance` is exactly + # that case (CMakeLists names tests/spec_decode_acceptance_test.py, not the tool). Measured while writing this: the + # narrower rule reported it as unrun, which is the false positive that would have made the ratchet noise. + regs += sorted((root / "tests").glob("*.py")) if (root / "tests").is_dir() else [] + for path in regs: + if not path.is_file(): + continue + text = path.read_text(errors="replace") + for name in need: + # a registration names the file (`tools/x.py --self-test`); a test that imports it names the STEM + # (`import x`) -- both run it, which is the question this asks + if name in text or pathlib.PurePosixPath(name).stem in text: + found[name].append(str(path.relative_to(root))) + return found + + def check(root): tests = root / "tests" if not tests.is_dir(): @@ -117,7 +166,12 @@ def check(root): for name in idle: print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing" % (name, ALLOW)) - if drift or gone or idle: + unrun = [n for n in tools_with_selftest(root) if not registered_tools(root, [n])[n] and n not in allow] + for name in unrun: + print("tests_referenced: tools/%s declares a self-test and nothing registers it -- add a ctest row " + "(add_python_test in CMakeLists.txt), or list it in %s with the reason its self-test cannot be a row" + % (name, ALLOW)) + if drift or gone or idle or unrun: return 1 print("tests_referenced: all %d test script(s) are referenced%s" % ( len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))))) @@ -181,7 +235,25 @@ def root_with(files): str(ALLOW): "tests/keeper2.sh # path form is accepted\n"}) cases.append(("a path-form allow entry is accepted", check(r), 0, None)) - # 8. no tests/ at all -> cannot tell, not "clean" + # 8. engine#1901: a tools/*.py that DECLARES a self-test and nothing registers + r = root_with({"tests/one.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\n", + "tools/orphan_self.py": "ok = True\nif '--self-test' in argv: pass\n"}) + cases.append(("a tool with an unregistered self-test is drift", check(r), 1, "tools/orphan_self.py")) + cases_root["a tool with an unregistered self-test is drift"] = r + # ... and the same tool once a registration names it is clean (so the arm above is not just "tools/ exists") + r = root_with({"tests/one.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\nadd_python_test(orphan tools/orphan_self.py --self-test)\n", + "tools/orphan_self.py": "ok = True\nif '--self-test' in argv: pass\n"}) + cases.append(("a registered self-test is not drift", check(r), 0, None)) + # ... and a tool a registered tests/*.py imports is RUN, even though no CMake line names the file + r = root_with({"tests/one.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\n", + "tests/uses_tool.py": "import sys; sys.path.insert(0, 'tools'); import orphan_self\n", + "tools/orphan_self.py": "ok = True\nif '--self-test' in argv: pass\n"}) + cases.append(("a tool a registered test imports is not drift", check(r), 0, None)) + + # 9. no tests/ at all -> cannot tell, not "clean" r = root_with({"CMakeLists.txt": ""}) cases.append(("no tests/ directory is 'cannot tell'", check(r), 2, None))