Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -871,6 +871,26 @@ if(Python3_Interpreter_FOUND)
# payloads under tests/fixtures/fflm/ — engine #588), so a regression cannot leave the
# daily alert permanently green.
add_test(NAME census_watch COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/census_watch_test.py)
# engine#2018: eight tests/*_test.py were on main with no ctest row at all, so nothing ran them -- #1824's
# sweep covers tests/*.sh and #1911's covers tools/*.py with --self-test, and the third file kind had no rule.
# Each of these passes when run by hand (2026-11-10, rc=0); the audit that keeps the set from growing is
# tools/ci/python_test_referenced_check.py. A row that runs Python names the interpreter rather than relying on
# the file's exec bit, and must sit after find_package(Python3) (tests/python_test_order_check.py).
add_test(NAME check_pins COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/check_pins_test.py)
add_test(NAME gguf_to_q4nx COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gguf_to_q4nx_test.py)
add_test(NAME hrx_long_prompt COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/hrx_long_prompt_test.py)
add_test(NAME hyperloom_benchmark COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py)
add_test(NAME moe_expert_chunks_accum COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/moe_expert_chunks_accum_test.py)
add_test(NAME mtp_draft_vocab COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/mtp_draft_vocab_test.py)
add_test(NAME repack_verify_source COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/repack_verify_source_test.py)
add_test(NAME ternary_to_q4_0 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/ternary_to_q4_0_test.py)
# engine#2018: the audit that keeps the eight rows above from being the last ones added. Fails when a
# tests/*_test.py is named by no row and excused by nothing, and when an excuse outlives the row it excused.
add_test(NAME python_tests_referenced
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/python_test_referenced_check.py --check)
add_test(NAME python_tests_referenced_self
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/python_test_referenced_check.py --self-test)

# an installed package must find its libraries beside itself: the 2026-11-08 incident's rule, from
# the installed side that no release gate can reach (engine #1411). Skips without patchelf/readelf.
add_test(NAME installed_rpath COMMAND bash ${CMAKE_SOURCE_DIR}/tests/installed_rpath_test.sh)
Expand Down
9 changes: 6 additions & 3 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,12 @@ an RFC or issue.
`onebit_npu_forward_syntax` is, and `tools/ci/ci_targets_check.py --check` keeps the count from growing. A
`tests/*.sh` script has to be named by a row — or listed with a reason in `tools/ci/unreferenced_tests.txt` if it is
not a row (`anchor_rerun.sh`, an operator measurement that always exits 0) — and `tools/ci/tests_referenced_check.py
--check` fails on one that is referenced by nothing. A row that runs Python goes through `add_python_test()`, which
hands the script to the interpreter instead of relying on the file's exec bit; `tests/python_test_order_check.py
--check` enforces that it follows `find_package(Python3)`. Two rows may not share a name: CMake stops at configure with
--check` fails on one that is referenced by nothing. A row that runs Python names the interpreter
explicitly (`add_test(NAME x COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/x_test.py)`) rather than relying on the
file's exec bit; `tests/python_test_order_check.py --check` enforces that every `${Python3_EXECUTABLE}` use follows
`find_package(Python3)`, and `tools/ci/python_test_referenced_check.py --check` fails when a `tests/*_test.py` has no row
and no recorded reason (`tools/ci/unreferenced_python_tests.txt`) — the third file kind after #1824's `tests/*.sh` and
#1911's `tools/*.py`. Two rows may not share a name: CMake stops at configure with
`add_test given test NAME "…" which already exists in this directory`.
- **A new test's registration goes in its own file.** Put the `add_test(...)` (or `add_python_test(...)`) row, its
comment, and its `set_tests_properties` in a new `cmake/tests-<name>.cmake` with the Apache notice, and do not touch
Expand Down
165 changes: 165 additions & 0 deletions tools/ci/python_test_referenced_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A `tests/*_test.py` with no ctest row is a file nothing runs (engine#2018).

The sweep covers three file kinds and this is the third:

- `tests/*.sh` -> tools/ci/tests_referenced_check.py (#1824)
- `tools/*.py` with `--self-test` -> tools/ci/selftest_registration_check.py (#1911)
- `tests/*_test.py` -> here: a row is `add_test(... COMMAND ${Python3_EXECUTABLE} <path> ...)`, or the file is listed
with a reason in tools/ci/unreferenced_python_tests.txt.

Both directions are checked, because an excuse that has outlived its reason is the same defect one step later: a name
in the allow file that IS registered now passes as "deliberately unrun", and the next reader believes it.

usage: tools/ci/python_test_referenced_check.py [--check | --self-test] [--root DIR]
"""

import argparse
import glob
import os
import re
import sys

ALLOW = "tools/ci/unreferenced_python_tests.txt"


def cmake_text(root):
"""Every CMake file a row can live in: the root list, the cmake/ registration files, and one level of subproject."""
out = []
for pattern in ("CMakeLists.txt", "cmake/*.cmake", "*/CMakeLists.txt"):
for p in sorted(glob.glob(os.path.join(root, pattern))):
try:
with open(p, encoding="utf-8", errors="replace") as f:
out.append((p, f.read()))
except OSError:
continue
return out


def python_tests(root):
return sorted(os.path.basename(p) for p in glob.glob(os.path.join(root, "tests/*_test.py")))


def excuses(root):
"""name -> reason, from the allow file. A line without a reason is refused: an unexplained excuse is a silencer."""
path = os.path.join(root, ALLOW)
out = {}
if not os.path.exists(path):
return out
bad = []
with open(path, encoding="utf-8") as f:
for n, line in enumerate(f, 1):
line = line.strip()
if not line or line.startswith("#"):
continue
name, sep, reason = line.partition(":")
if not sep or not reason.strip():
bad.append((n, line))
continue
out[name.strip()] = reason.strip()
if bad:
raise SystemExit("python_test_referenced_check: %s line(s) with no reason: %s"
% (len(bad), ", ".join(str(b[0]) for b in bad)))
return out


def check(root, quiet=False):
tests = python_tests(root)
text = "\n".join(t for _, t in cmake_text(root))
allow = excuses(root)
unrun, stale = [], []
for t in tests:
named = t in text
if not named and t not in allow:
unrun.append(t)
elif named and t in allow:
stale.append(t)
for t in unrun:
print(" UNRUN %s: no add_test row names it, and %s does not excuse it" % (t, ALLOW), file=sys.stderr)
for t in stale:
print(" STALE %s: excused in %s but a row DOES name it -- drop the excuse" % (t, ALLOW), file=sys.stderr)
if not quiet:
print("python_test_referenced_check: %d test(s), %d run by a row, %d excused, %d unrun, %d stale"
% (len(tests), len(tests) - len(unrun) - len(allow) + len(stale), len(allow) - len(stale), len(unrun), len(stale)))
return 0 if not unrun and not stale else 1


def write(path, text):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)


def self_test():
import tempfile
ok = True
with tempfile.TemporaryDirectory() as d:
write(os.path.join(d, "CMakeLists.txt"),
"add_test(NAME a COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/a_test.py)\n")
write(os.path.join(d, "tests/a_test.py"), "")
write(os.path.join(d, "tests/b_test.py"), "")
write(os.path.join(d, "tests/c_test.py"), "")
# b is unrun with no excuse -> must fail; c is excused -> must pass; nothing stale yet
r = check(d, quiet=True)
if r == 0:
print(" FAIL an unrun test with no excuse passed the check", file=sys.stderr)
ok = False
else:
print(" ok an unrun test with no excuse fails the check")
write(os.path.join(d, ALLOW), "# name: reason\nb_test.py: run by hand, needs a 27B pack on disk\n"
"c_test.py: needs the pinned llama.cpp checkout\n")
if check(d, quiet=True) != 0:
print(" FAIL an excused test did not clear the check", file=sys.stderr)
ok = False
else:
print(" ok an excused test clears the check")
# the other direction: an excuse for a file that IS registered is stale
write(os.path.join(d, ALLOW), "a_test.py: nobody needs this\nb_test.py: needs a pack\n"
"c_test.py: needs the pinned llama.cpp checkout\n")
if check(d, quiet=True) == 0:
print(" FAIL an excuse for a registered test did not fail as stale", file=sys.stderr)
ok = False
else:
print(" ok an excuse that outlived its reason fails as stale")
# an excuse with no reason is refused rather than silently silencing a test
write(os.path.join(d, ALLOW), "b_test.py:\n")
try:
excuses(d)
print(" FAIL a reasonless excuse was accepted", file=sys.stderr)
ok = False
except SystemExit:
print(" ok a reasonless excuse is refused")
print("python_test_referenced_check self-test: %s" % ("ok" if ok else "FAILED"))
return 0 if ok else 1


def main():
ap = argparse.ArgumentParser()
ap.add_argument("--check", action="store_true")
ap.add_argument("--self-test", action="store_true")
ap.add_argument("--root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
a = ap.parse_args()
if a.self_test:
return self_test()
if a.check:
return check(a.root)
print(__doc__.strip().splitlines()[-1], file=sys.stderr)
return 2


if __name__ == "__main__":
sys.exit(main())
Loading