From 14e8d1ef04824d86d178e13cafafe62857fd0e66 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 08:02:14 -0400 Subject: [PATCH] tools: register the self-tests nothing ran, and teach the sweep to see a tools/*.py at all (fixes #1901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #1824's check asks whether every `tests/*.sh` is referenced by something that can run it -- `need` is that glob and nothing else, so a tool shipping a `--self-test` was invisible to it. Two were unregistered at the time the issue was filed; the rule below finds **three**, and the third is the reason the rule exists rather than the two hand-picked files: ``` tools/batched_repeat.py --selftest (not in the issue's list) tools/whisper_encoder_baseline.py --self-test ``` Both are registered here through #1891's `add_python_test()`: `batched_repeat.py --selftest` (its own spelling, since before the hyphenated convention) and `whisper_encoder_baseline.py --self-test`. Both pass on this tree, device-free. **The rule, and why it is narrow.** `need` now also includes any `tools/*.py` that *declares* a self-test. The blanket version ("every tools/*.py referenced by nothing") found 16 of 60 on this tree -- most legitimately CLI-only -- and a list that noisy is one nobody reads. A tool that ships a check and is registered nowhere is the defect; a tool that only has a CLI is not. **And the sweep was green on a disabled registration**, which is the same defect one layer in. `referenced()` counted a basename anywhere in a place that can run things, so `# add_python_test(...)` — or `# add_test(...)` for the 73 scripts — read as a runner. Comment-only lines are dropped now, and the mutation shows it: with both of this PR's registrations commented out the sweep names them instead of passing. ## Self-test Two cases added, in the file's own style (synthetic roots, and the verdict has to NAME the offender): * a self-testing tool nothing registers → drift, named; * the same tool registered → clean, **and** a `tools/*.py` with no self-test is not a row (the negative that keeps the rule narrow). ## Verified ``` tools/ci/tests_referenced_check.py --check --root . -> 73 script(s) and 17 self-testing tool(s) are referenced (rc 0) tools/ci/tests_referenced_check.py --self-test -> 0 failure(s) python3 tools/batched_repeat.py --selftest -> selftest: ok python3 tools/whisper_encoder_baseline.py --self-test -> whisper_encoder_baseline: ok ctest -R 'tests_referenced|tool_selftest|ci_targets' -> 100% tests passed, 0 failed out of 6 mutation: comment out both registrations -> the sweep names both tools tools/copyright.py --check (staged) -> rc 0 ``` Fixes #1901. Refs #1824, #1891. --- CMakeLists.txt | 5 ++++ tools/ci/tests_referenced_check.py | 47 ++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 636a3d5c2..3480564a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,6 +318,11 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. +# engine#1901: two tools shipped a self-test and nothing ran them -- the #1824 sweep asks about tests/*.sh only, so +# a tool's own check was invisible to it (the same defect one file kind over). Registered here, and the sweep now +# covers tools/*.py that declare one. `batched_repeat.py` spells it --selftest, `whisper_encoder_baseline.py` --self-test. +add_python_test(tool_selftest_batched_repeat ${CMAKE_SOURCE_DIR}/tools/batched_repeat.py --selftest) +add_python_test(tool_selftest_whisper_encoder ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 90ba398f8..1fc6ba518 100755 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -85,10 +85,22 @@ def scan_files(root): continue +def uncommented(text): + """The text with comment-only lines dropped: a registration commented out does not run anything. + + engine#1901: the rule was "mentions its basename somewhere that can run things", which counted a `# add_test(...)` + as a runner -- so commenting a registration out left the sweep green (measured: commenting out both of this PR's + registrations changed nothing). Trailing comments are left alone; a line whose first non-space character is `#` is + the shape a disabled registration takes, in CMake, shell, YAML and Python alike. + """ + return "\n".join(l for l in text.split("\n") if not l.lstrip().startswith("#")) + + def referenced(root, need): """{name: [files]}, ignoring a file mentioning its own basename.""" found = {name: [] for name in need} for path, text in scan_files(root): + text = uncommented(text) for name in need: if name not in text: continue @@ -107,20 +119,34 @@ def check(root): allow = allow_list(root) found = referenced(root, need) drift = [n for n in need if not found[n] and n not in allow] + # engine#1901: the same defect one file kind over -- a tool that ships a --self-test and is registered nowhere is a + # check nobody runs, and this sweep could not see it because `need` is tests/*.sh only. The rule is narrow on purpose + # ("DECLARES a self-test"): the blanket "tools/*.py referenced by nothing" found 16 of 60 on this tree, most of them + # legitimately CLI-only, and a list that noisy gets ignored. Both spellings count -- tools/mtp_head_ref.py has had + # --selftest since before the hyphenated convention. + tools_need = sorted(p.name for p in (root / "tools").glob("*.py") + if "--self-test" in p.read_text(errors="replace") + or "--selftest" in p.read_text(errors="replace")) + found_tools = referenced(root, tools_need) + tools_drift = [n for n in tools_need if not found_tools[n] and n not in allow] gone = [n for n in allow if n not in need] idle = [n for n in need if n in allow and found[n]] for name in drift: print("tests_referenced: %s is referenced by nothing that can run it -- register it (add_test in " "CMakeLists.txt or cmake/*.cmake), or list it in %s with the reason it must not be a row" % (name, ALLOW)) + for name in tools_drift: + print("tests_referenced: tools/%s declares a self-test and nothing registers it -- add_python_test in " + "CMakeLists.txt, or list it in %s with the reason" % (name, ALLOW)) for name in gone: print("tests_referenced: %s is in %s but no such test exists any more -- drop the line" % (name, ALLOW)) for name in idle: print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing" % (name, ALLOW)) - if drift or gone or idle: + if drift or gone or idle or tools_drift: return 1 - print("tests_referenced: all %d test script(s) are referenced%s" % ( - len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))))) + print("tests_referenced: all %d test script(s) and %d self-testing tool(s) are referenced%s" % ( + len(need), len(tools_need), + "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))))) return 0 @@ -147,6 +173,21 @@ def root_with(files): r = root_with({"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n"}) cases.append(("a registration in CMakeLists.txt counts", check(r), 0, None)) + # 2b. engine#1901: a tool that DECLARES a self-test and is registered nowhere -> drift, and it is NAMED + r = root_with({"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n", + "tools/orphan.py": "ap.add_argument('--self-test')\n", + "tools/library_only.py": "def f():\n pass\n"}) + cases.append(("a self-testing tool nothing registers is drift", check(r), 1, "orphan.py")) + cases_root["a self-testing tool nothing registers is drift"] = r + # 2c. the same tool registered -> clean, and a tool with NO self-test is never a row (the blanket sweep is noise) + r = root_with({"tests/ok.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n" + "add_python_test(t tool.py --self-test)\n", + "tools/tool.py": "ap.add_argument('--self-test')\n", + "tools/library_only.py": "def f():\n pass\n"}) + cases.append(("a registered self-testing tool is clean, and a tool with none is not a row", check(r), 0, None)) + cases_root["a registered self-testing tool is clean, and a tool with none is not a row"] = r + # 3. referenced from an included family file -> clean r = root_with({"tests/fam.sh": "#!/bin/sh\n", "CMakeLists.txt": "", "cmake/tests-family.cmake": "add_test(NAME fam COMMAND bash tests/fam.sh)\n"})