diff --git a/CMakeLists.txt b/CMakeLists.txt index 636a3d5c2..050a9c8c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,6 +304,18 @@ add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py # tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink. add_python_test(tests_referenced ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(tests_referenced_self ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --self-test) +# engine#1901: five tools ship a --self-test that no runner named, so the arms existed, were maintained beside the logic they +# check, and never executed. All five run device-free and fast (58-277 ms measured), so they are rows rather than allow-list +# excuses, and tools/ci/tests_referenced_check.py now counts tools/**/*.py that declare one and are named by no RUNNER +# (CMakeLists/cmake/workflows -- a citation in another tool's source is not a runner: that is the 5-vs-2 difference). +# Registered through add_python_test (engine#1864), so they run THROUGH the interpreter: my first version used the raw +# `${Python3_EXECUTABLE}` form above the find_package and the repository's own ordering guard caught it by name -- +# tools/mtp_head_ref.py is the one of the five without an exec bit, and it became BAD_COMMAND under ctest. +add_python_test(tool_selftest_batched_repeat ${CMAKE_SOURCE_DIR}/tools/batched_repeat.py --selftest) +add_python_test(tool_selftest_logprob_agree ${CMAKE_SOURCE_DIR}/tools/logprob_agree.py --selftest) +add_python_test(tool_selftest_mtp_head_ref ${CMAKE_SOURCE_DIR}/tools/mtp_head_ref.py --selftest) +add_python_test(tool_selftest_spec_decode_acceptance ${CMAKE_SOURCE_DIR}/tools/spec_decode_acceptance.py --selftest) +add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test) # engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820). add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh) # engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 90ba398f8..2df91ec28 100755 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -22,6 +22,11 @@ reads as green for ever -- the same defect as #1595, one file kind over. need = every tests/*.sh + AND every tools/**/*.py that declares a --self-test (engine#1901): a tool that ships one is claiming it can be + verified without its real inputs, so an unregistered one is an arm nobody runs. For those the corpus is + RUNNERS only (CMakeLists.txt, cmake/, .github/workflows/) -- a mention in another tool's source is a citation, + not a runner. That distinction is the count: 2 with the wide corpus, 5 with this one, and 5 is the honest + number because `tools/mtp_head_ref.py` is only *cited* by two other tools. have = every file that MENTIONS its basename, in a place that can run things: CMakeLists.txt, cmake/**, scripts/**, tools/**, tests/**, .github/workflows/** drift = need - have - allow-list @@ -42,6 +47,7 @@ import argparse import pathlib +import re import sys ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt") @@ -98,6 +104,37 @@ def referenced(root, need): return found +SELFTEST = re.compile(r"--self[-_]?test\b") +RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows") + + +def unregistered_selftest_tools(root): + """tools/**/*.py that declare a self-test and that no RUNNER names (engine#1901).""" + tools = root / "tools" + if not tools.is_dir(): + return [] + runner_text = [] + for sub in RUNNERS: + p = root / sub + files = [p] if p.is_file() else (sorted(q for q in p.rglob("*") if q.is_file()) if p.is_dir() else []) + for f in files: + try: + runner_text.append(f.read_text(errors="replace")) + except OSError: + pass + drift = [] + for tool in sorted(tools.rglob("*.py")): + try: + src = tool.read_text(errors="replace") + except OSError: + continue + if not SELFTEST.search(src): + continue + if not any(tool.name in text for text in runner_text): + drift.append(str(tool.relative_to(root))) + return drift + + def check(root): tests = root / "tests" if not tests.is_dir(): @@ -117,13 +154,23 @@ def check(root): for name in idle: print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing" % (name, ALLOW)) - if drift or gone or idle: + tools_drift = unregistered_selftest_tools(root) + for rel in tools_drift: + print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in " + "CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel) + if drift or gone or idle or tools_drift: return 1 - print("tests_referenced: all %d test script(s) are referenced%s" % ( - len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))))) + print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % ( + len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))), + len(_selftest_tools(root)))) return 0 +def _selftest_tools(root): + tools = root / "tools" + return [p for p in tools.rglob("*.py") if SELFTEST.search(p.read_text(errors="replace"))] if tools.is_dir() else [] + + def self_test(): """Synthetic roots: a check that only ever sees a good tree proves nothing.""" import tempfile @@ -176,6 +223,22 @@ def root_with(files): str(ALLOW): "one.sh # but it is registered\n"}) cases.append(("an allow entry that suppresses nothing fails", check(r), 1, "one.sh")); cases_root["an allow entry that suppresses nothing fails"] = r + # 9b. engine#1901: a tool that declares a self-test and is named by no RUNNER is reported, and a mention in + # another tool's source does not count as one -- that distinction is the difference between 2 and 5 on main. + base_ok = {"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND tests/ok.sh)\n"} + r = root_with(dict(base_ok, **{"tools/quiet.py": "import sys\n# --self-test\n"})) + cases.append(("a tool self-test no runner names is drift", check(r), 1, "quiet.py")) + cases_root["a tool self-test no runner names is drift"] = r + r2 = root_with(dict(base_ok, **{"tools/cited.py": "# --self-test\n", + "tools/other.py": "see tools/cited.py for the arm\n"})) + cases.append(("a citation in another tool is not a runner", check(r2), 1, "cited.py")) + cases_root["a citation in another tool is not a runner"] = r2 + r3 = root_with({"tests/ok.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME ok COMMAND tests/ok.sh)\n" + "add_test(NAME t COMMAND tools/registered.py --self-test)\n", + "tools/registered.py": "# --self-test\n"}) + cases.append(("a registered tool self-test is clean", check(r3), 0, None)) + # 9. the entry may be written either way: path form and basename mean the same line r = root_with({"tests/keeper2.sh": "#!/bin/sh\n", "CMakeLists.txt": "", str(ALLOW): "tests/keeper2.sh # path form is accepted\n"})