Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,15 @@ add_test(NAME kernel_set_check COMMAND bash ${CMAKE_SOURCE_DIR}/tests/kernel_set
# gets one row.
add_test(NAME dupdef_check COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/dupdef_check.py --check
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
# engine#2456: `tests_referenced_check` used to accept a MENTION of a script as an invocation, so four tools whose
# `--self-test`/`--selftest` no row ever ran read as referenced -- by a comment in a test file. It now requires the
# script's name and a self-test flag on the same line, which is what these four rows are: the check got stricter and
# main has to be green under the stricter rule. Each flag spelling is the tool's own (`gsm8k_eval` spells it without
# the hyphen; the other three with it).
add_test(NAME dupdef_check_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/dupdef_check.py --self-test)
add_test(NAME gsm8k_eval_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/gsm8k_eval.py --selftest)
add_test(NAME q4nx_diff_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/q4nx_diff.py --self-test)
add_test(NAME seal_manifest_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/seal_manifest.py --self-test)
# engine#2255: `--help` must not be a traceback. Five tools indexed sys.argv directly, so the flag
# arrived as a filename -- one with IndexError, the others with `FileNotFoundError: '--help'`, and two
# of those on `--help/dd_metastate_...`, which is what a path join with the flag looks like. Fixed; this
Expand Down
37 changes: 36 additions & 1 deletion tools/ci/tests_referenced_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,13 @@ def referenced(root, need):


SELFTEST = re.compile(r"--self[-_]?test\b")

# engine#2456: a runner line that only *documents* a script is not a runner. Both comment styles appear in the tree
# (`#` in CMake/shell/YAML, `//` in C++); a line starting with either, after leading whitespace, cannot invoke
# anything, so a match there is a mention.
def is_comment(line):
t = line.lstrip()
return t.startswith("#") or t.startswith("//")
RUNNERS = ("CMakeLists.txt", "cmake", ".github/workflows")
# A tool's `--self-test` can also be run from a python test, so THAT gate reads one more directory. The python-test gate must
# not: with "tests" in the shared tuple, `tests/lonely_test.py` mentioning its own name satisfied "some runner names it" and the
Expand All @@ -179,6 +186,12 @@ def unregistered_selftest_tools(root):
cand += [p for p in scripts.glob("*.sh")]
if not cand:
return []
# engine#2456: a MENTION is not an INVOCATION. `tests/` is a runner directory, so a comment in a test file that
# names the script used to satisfy this check -- i.e. the class it exists to prevent ("a self-test nobody runs",
# engine#2429/#2418/#2416) could be re-introduced by documenting the script. Reproduced on #2455: a comment in
# tests/pack_modes_test.sh naming scripts/window-run.sh read green while no row existed, and rewording the
# comment read red -- the check was answering "does this string appear" where it meant "does a runner run it".
# So collect LINES, not blobs, and count only a line that names the script AND passes --self-test.
runner_text = []
for sub in TOOL_RUNNERS:
p = root / sub
Expand All @@ -188,6 +201,7 @@ def unregistered_selftest_tools(root):
runner_text.append(f.read_text(errors="replace"))
except OSError:
pass
runner_lines = [ln for text in runner_text for ln in text.splitlines()]
drift = []
for tool in sorted(cand):
try:
Expand All @@ -196,7 +210,12 @@ def unregistered_selftest_tools(root):
continue
if not SELFTEST.search(src):
continue
if not any(tool.name in text for text in runner_text):
# Both spellings occur in the tree (`--self-test` and `--selftest`); the same regex the declaration side
# uses must decide the invocation side, or a tool declaring one form reads as unreferenced forever.
# AND the line must not be a comment: a comment naming the script WITH the flag is precisely the case that
# read green for months (`# see tools/x.py --self-test for the arms`), and it is the shape the self-test's
# first arm pins -- requiring the flag alone did not catch it, excluding comment lines does.
if not any(tool.name in ln and SELFTEST.search(ln) and not is_comment(ln) for ln in runner_lines):
drift.append(str(tool.relative_to(root)))
return drift

Expand Down Expand Up @@ -364,6 +383,22 @@ def root_with(files):
".github/workflows/ci.yml": "run: bash tests/ci.sh\n"})
cases.append(("a mention in a workflow counts", check(r), 0, None))

# 4a. engine#2456: a MENTION is not an INVOCATION. A comment in a runner file that names the script used to
# satisfy this check -- reproduced on #2455, where a comment in tests/pack_modes_test.sh naming
# scripts/window-run.sh read green while no row existed. The pair below is the whole rule: the same tree, once
# with the name only in a comment (drift) and once with a real row (clean), so a regression to substring
# matching fails the first arm while anything that only checks the positive case still passes.
tool = "import sys\nif '--self-test' in sys.argv: pass\n"
row_t = "add_test(NAME t COMMAND bash tests/t.sh)\n"
r = root_with({"tools/mine.py": tool,
"tests/t.sh": "#!/bin/sh\n# see tools/mine.py --self-test for the arms\n",
"CMakeLists.txt": row_t})
cases.append(("a COMMENT naming a tool --self-test is still drift", check(r), 1, "tools/mine.py"))
cases_root["a COMMENT naming a tool --self-test is still drift"] = r
r = root_with({"tools/mine.py": tool, "tests/t.sh": "#!/bin/sh\n",
"CMakeLists.txt": row_t + "add_test(NAME mine COMMAND python3 tools/mine.py --self-test)\n"})
cases.append(("...and a real add_test row clears the same tree", check(r), 0, None))

# 4b. a PYTHON test naming only itself is drift even though `tests/` is a run site for TOOL self-tests -- the two gates
# read different corpora on purpose (measured: sharing one tuple let tests/lonely_test.py register itself)
r = root_with({"tests/lonely_test.py": "import sys\n# usage: tests/lonely_test.py\nprint('--self-test')\n",
Expand Down
Loading