Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,18 @@ add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py
# tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink.
add_python_test(tests_referenced ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --check --root ${CMAKE_SOURCE_DIR})
add_python_test(tests_referenced_self ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --self-test)
# engine#1901: five tools ship a --self-test that no runner named, so the arms existed, were maintained beside the logic they
# check, and never executed. All five run device-free and fast (58-277 ms measured), so they are rows rather than allow-list
# excuses, and tools/ci/tests_referenced_check.py now counts tools/**/*.py that declare one and are named by no RUNNER
# (CMakeLists/cmake/workflows -- a citation in another tool's source is not a runner: that is the 5-vs-2 difference).
# Registered through add_python_test (engine#1864), so they run THROUGH the interpreter: my first version used the raw
# `${Python3_EXECUTABLE}` form above the find_package and the repository's own ordering guard caught it by name --
# tools/mtp_head_ref.py is the one of the five without an exec bit, and it became BAD_COMMAND under ctest.
add_python_test(tool_selftest_batched_repeat ${CMAKE_SOURCE_DIR}/tools/batched_repeat.py --selftest)
add_python_test(tool_selftest_logprob_agree ${CMAKE_SOURCE_DIR}/tools/logprob_agree.py --selftest)
add_python_test(tool_selftest_mtp_head_ref ${CMAKE_SOURCE_DIR}/tools/mtp_head_ref.py --selftest)
add_python_test(tool_selftest_spec_decode_acceptance ${CMAKE_SOURCE_DIR}/tools/spec_decode_acceptance.py --selftest)
add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test)
# engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820).
add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh)
# engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three
Expand Down
69 changes: 66 additions & 3 deletions tools/ci/tests_referenced_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@
reads as green for ever -- the same defect as #1595, one file kind over.

need = every tests/*.sh
AND every tools/**/*.py that declares a --self-test (engine#1901): a tool that ships one is claiming it can be
verified without its real inputs, so an unregistered one is an arm nobody runs. For those the corpus is
RUNNERS only (CMakeLists.txt, cmake/, .github/workflows/) -- a mention in another tool's source is a citation,
not a runner. That distinction is the count: 2 with the wide corpus, 5 with this one, and 5 is the honest
number because `tools/mtp_head_ref.py` is only *cited* by two other tools.
have = every file that MENTIONS its basename, in a place that can run things:
CMakeLists.txt, cmake/**, scripts/**, tools/**, tests/**, .github/workflows/**
drift = need - have - allow-list
Expand All @@ -42,6 +47,7 @@

import argparse
import pathlib
import re
import sys

ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt")
Expand Down Expand Up @@ -98,6 +104,37 @@ def referenced(root, need):
return found


SELFTEST = re.compile(r"--self[-_]?test\b")
RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows")


def unregistered_selftest_tools(root):
"""tools/**/*.py that declare a self-test and that no RUNNER names (engine#1901)."""
tools = root / "tools"
if not tools.is_dir():
return []
runner_text = []
for sub in RUNNERS:
p = root / sub
files = [p] if p.is_file() else (sorted(q for q in p.rglob("*") if q.is_file()) if p.is_dir() else [])
for f in files:
try:
runner_text.append(f.read_text(errors="replace"))
except OSError:
pass
drift = []
for tool in sorted(tools.rglob("*.py")):
try:
src = tool.read_text(errors="replace")
except OSError:
continue
if not SELFTEST.search(src):
continue
if not any(tool.name in text for text in runner_text):
drift.append(str(tool.relative_to(root)))
return drift


def check(root):
tests = root / "tests"
if not tests.is_dir():
Expand All @@ -117,13 +154,23 @@ def check(root):
for name in idle:
print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing"
% (name, ALLOW))
if drift or gone or idle:
tools_drift = unregistered_selftest_tools(root)
for rel in tools_drift:
print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in "
"CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel)
if drift or gone or idle or tools_drift:
return 1
print("tests_referenced: all %d test script(s) are referenced%s" % (
len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow)))))
print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % (
len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))),
len(_selftest_tools(root))))
return 0


def _selftest_tools(root):
tools = root / "tools"
return [p for p in tools.rglob("*.py") if SELFTEST.search(p.read_text(errors="replace"))] if tools.is_dir() else []


def self_test():
"""Synthetic roots: a check that only ever sees a good tree proves nothing."""
import tempfile
Expand Down Expand Up @@ -176,6 +223,22 @@ def root_with(files):
str(ALLOW): "one.sh # but it is registered\n"})
cases.append(("an allow entry that suppresses nothing fails", check(r), 1, "one.sh")); cases_root["an allow entry that suppresses nothing fails"] = r

# 9b. engine#1901: a tool that declares a self-test and is named by no RUNNER is reported, and a mention in
# another tool's source does not count as one -- that distinction is the difference between 2 and 5 on main.
base_ok = {"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND tests/ok.sh)\n"}
r = root_with(dict(base_ok, **{"tools/quiet.py": "import sys\n# --self-test\n"}))
cases.append(("a tool self-test no runner names is drift", check(r), 1, "quiet.py"))
cases_root["a tool self-test no runner names is drift"] = r
r2 = root_with(dict(base_ok, **{"tools/cited.py": "# --self-test\n",
"tools/other.py": "see tools/cited.py for the arm\n"}))
cases.append(("a citation in another tool is not a runner", check(r2), 1, "cited.py"))
cases_root["a citation in another tool is not a runner"] = r2
r3 = root_with({"tests/ok.sh": "#!/bin/sh\n",
"CMakeLists.txt": "add_test(NAME ok COMMAND tests/ok.sh)\n"
"add_test(NAME t COMMAND tools/registered.py --self-test)\n",
"tools/registered.py": "# --self-test\n"})
cases.append(("a registered tool self-test is clean", check(r3), 0, None))

# 9. the entry may be written either way: path form and basename mean the same line
r = root_with({"tests/keeper2.sh": "#!/bin/sh\n", "CMakeLists.txt": "",
str(ALLOW): "tests/keeper2.sh # path form is accepted\n"})
Expand Down
Loading