From b5646f05b341c2236f2b55ec81757d274dfcccae Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 17:12:43 -0400 Subject: [PATCH] tests: register the eight tests/*_test.py nothing ran, and audit the third file kind (engine#2018) Measured on main 2026-11-10: of 41 tests/*_test.py on disk, 8 were named by no CMake file at all -- so nothing ran them and nothing noticed: check_pins_test gguf_to_q4nx_test hrx_long_prompt_test hyperloom_benchmark_test moe_expert_chunks_accum_test mtp_draft_vocab_test repack_verify_source_test ternary_to_q4_0_test All eight pass when run by hand (rc=0, 2026-11-10), so they are registered as rows rather than excused: add_test(NAME x COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/x_test.py), after find_package(Python3) as tests/python_test_order_check.py requires. 41 of 41 are now run by a row. tools/ci/python_test_referenced_check.py is the audit that keeps the set from growing, and it checks BOTH directions: an unrun test with no recorded reason fails (UNRUN), and an excuse in tools/ci/unreferenced_python_tests.txt that outlives its reason fails too (STALE) -- an excuse for a file that a row does name passes as "deliberately unrun" and the next reader believes it. A line in the allow file with no reason after the colon is refused outright rather than silently silencing a test. The self-test pins all four arms on synthetic trees (its first version had an unexcused fixture file, which its own second arm caught). CONTRIBUTING.md documented `add_python_test()` for this, and no such macro exists anywhere in the tree -- the 41 rows all name ${Python3_EXECUTABLE} by hand. The rule now says what the rows do, and names the new check. Checks: python_test_order_check.py --check ok; python_test_referenced_check.py --check "41 test(s), 41 run by a row, 0 excused, 0 unrun, 0 stale"; --self-test 4/4; copyright rc=0. --- CMakeLists.txt | 20 +++ CONTRIBUTING.md | 9 +- tools/ci/python_test_referenced_check.py | 165 +++++++++++++++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 tools/ci/python_test_referenced_check.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b6821e3e..35b8d70a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -871,6 +871,26 @@ if(Python3_Interpreter_FOUND) # payloads under tests/fixtures/fflm/ — engine #588), so a regression cannot leave the # daily alert permanently green. add_test(NAME census_watch COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/census_watch_test.py) + # engine#2018: eight tests/*_test.py were on main with no ctest row at all, so nothing ran them -- #1824's + # sweep covers tests/*.sh and #1911's covers tools/*.py with --self-test, and the third file kind had no rule. + # Each of these passes when run by hand (2026-11-10, rc=0); the audit that keeps the set from growing is + # tools/ci/python_test_referenced_check.py. A row that runs Python names the interpreter rather than relying on + # the file's exec bit, and must sit after find_package(Python3) (tests/python_test_order_check.py). + add_test(NAME check_pins COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/check_pins_test.py) + add_test(NAME gguf_to_q4nx COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gguf_to_q4nx_test.py) + add_test(NAME hrx_long_prompt COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/hrx_long_prompt_test.py) + add_test(NAME hyperloom_benchmark COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py) + add_test(NAME moe_expert_chunks_accum COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/moe_expert_chunks_accum_test.py) + add_test(NAME mtp_draft_vocab COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/mtp_draft_vocab_test.py) + add_test(NAME repack_verify_source COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/repack_verify_source_test.py) + add_test(NAME ternary_to_q4_0 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/ternary_to_q4_0_test.py) + # engine#2018: the audit that keeps the eight rows above from being the last ones added. Fails when a + # tests/*_test.py is named by no row and excused by nothing, and when an excuse outlives the row it excused. + add_test(NAME python_tests_referenced + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/python_test_referenced_check.py --check) + add_test(NAME python_tests_referenced_self + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/python_test_referenced_check.py --self-test) + # an installed package must find its libraries beside itself: the 2026-11-08 incident's rule, from # the installed side that no release gate can reach (engine #1411). Skips without patchelf/readelf. add_test(NAME installed_rpath COMMAND bash ${CMAKE_SOURCE_DIR}/tests/installed_rpath_test.sh) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b86b8a268..1a2ea9a78 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,9 +83,12 @@ an RFC or issue. `onebit_npu_forward_syntax` is, and `tools/ci/ci_targets_check.py --check` keeps the count from growing. A `tests/*.sh` script has to be named by a row — or listed with a reason in `tools/ci/unreferenced_tests.txt` if it is not a row (`anchor_rerun.sh`, an operator measurement that always exits 0) — and `tools/ci/tests_referenced_check.py - --check` fails on one that is referenced by nothing. A row that runs Python goes through `add_python_test()`, which - hands the script to the interpreter instead of relying on the file's exec bit; `tests/python_test_order_check.py - --check` enforces that it follows `find_package(Python3)`. Two rows may not share a name: CMake stops at configure with + --check` fails on one that is referenced by nothing. A row that runs Python names the interpreter + explicitly (`add_test(NAME x COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/x_test.py)`) rather than relying on the + file's exec bit; `tests/python_test_order_check.py --check` enforces that every `${Python3_EXECUTABLE}` use follows + `find_package(Python3)`, and `tools/ci/python_test_referenced_check.py --check` fails when a `tests/*_test.py` has no row + and no recorded reason (`tools/ci/unreferenced_python_tests.txt`) — the third file kind after #1824's `tests/*.sh` and + #1911's `tools/*.py`. Two rows may not share a name: CMake stops at configure with `add_test given test NAME "…" which already exists in this directory`. - **A new test's registration goes in its own file.** Put the `add_test(...)` (or `add_python_test(...)`) row, its comment, and its `set_tests_properties` in a new `cmake/tests-.cmake` with the Apache notice, and do not touch diff --git a/tools/ci/python_test_referenced_check.py b/tools/ci/python_test_referenced_check.py new file mode 100644 index 000000000..68429b27a --- /dev/null +++ b/tools/ci/python_test_referenced_check.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A `tests/*_test.py` with no ctest row is a file nothing runs (engine#2018). + +The sweep covers three file kinds and this is the third: + +- `tests/*.sh` -> tools/ci/tests_referenced_check.py (#1824) +- `tools/*.py` with `--self-test` -> tools/ci/selftest_registration_check.py (#1911) +- `tests/*_test.py` -> here: a row is `add_test(... COMMAND ${Python3_EXECUTABLE} ...)`, or the file is listed + with a reason in tools/ci/unreferenced_python_tests.txt. + +Both directions are checked, because an excuse that has outlived its reason is the same defect one step later: a name +in the allow file that IS registered now passes as "deliberately unrun", and the next reader believes it. + +usage: tools/ci/python_test_referenced_check.py [--check | --self-test] [--root DIR] +""" + +import argparse +import glob +import os +import re +import sys + +ALLOW = "tools/ci/unreferenced_python_tests.txt" + + +def cmake_text(root): + """Every CMake file a row can live in: the root list, the cmake/ registration files, and one level of subproject.""" + out = [] + for pattern in ("CMakeLists.txt", "cmake/*.cmake", "*/CMakeLists.txt"): + for p in sorted(glob.glob(os.path.join(root, pattern))): + try: + with open(p, encoding="utf-8", errors="replace") as f: + out.append((p, f.read())) + except OSError: + continue + return out + + +def python_tests(root): + return sorted(os.path.basename(p) for p in glob.glob(os.path.join(root, "tests/*_test.py"))) + + +def excuses(root): + """name -> reason, from the allow file. A line without a reason is refused: an unexplained excuse is a silencer.""" + path = os.path.join(root, ALLOW) + out = {} + if not os.path.exists(path): + return out + bad = [] + with open(path, encoding="utf-8") as f: + for n, line in enumerate(f, 1): + line = line.strip() + if not line or line.startswith("#"): + continue + name, sep, reason = line.partition(":") + if not sep or not reason.strip(): + bad.append((n, line)) + continue + out[name.strip()] = reason.strip() + if bad: + raise SystemExit("python_test_referenced_check: %s line(s) with no reason: %s" + % (len(bad), ", ".join(str(b[0]) for b in bad))) + return out + + +def check(root, quiet=False): + tests = python_tests(root) + text = "\n".join(t for _, t in cmake_text(root)) + allow = excuses(root) + unrun, stale = [], [] + for t in tests: + named = t in text + if not named and t not in allow: + unrun.append(t) + elif named and t in allow: + stale.append(t) + for t in unrun: + print(" UNRUN %s: no add_test row names it, and %s does not excuse it" % (t, ALLOW), file=sys.stderr) + for t in stale: + print(" STALE %s: excused in %s but a row DOES name it -- drop the excuse" % (t, ALLOW), file=sys.stderr) + if not quiet: + print("python_test_referenced_check: %d test(s), %d run by a row, %d excused, %d unrun, %d stale" + % (len(tests), len(tests) - len(unrun) - len(allow) + len(stale), len(allow) - len(stale), len(unrun), len(stale))) + return 0 if not unrun and not stale else 1 + + +def write(path, text): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(text) + + +def self_test(): + import tempfile + ok = True + with tempfile.TemporaryDirectory() as d: + write(os.path.join(d, "CMakeLists.txt"), + "add_test(NAME a COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/a_test.py)\n") + write(os.path.join(d, "tests/a_test.py"), "") + write(os.path.join(d, "tests/b_test.py"), "") + write(os.path.join(d, "tests/c_test.py"), "") + # b is unrun with no excuse -> must fail; c is excused -> must pass; nothing stale yet + r = check(d, quiet=True) + if r == 0: + print(" FAIL an unrun test with no excuse passed the check", file=sys.stderr) + ok = False + else: + print(" ok an unrun test with no excuse fails the check") + write(os.path.join(d, ALLOW), "# name: reason\nb_test.py: run by hand, needs a 27B pack on disk\n" + "c_test.py: needs the pinned llama.cpp checkout\n") + if check(d, quiet=True) != 0: + print(" FAIL an excused test did not clear the check", file=sys.stderr) + ok = False + else: + print(" ok an excused test clears the check") + # the other direction: an excuse for a file that IS registered is stale + write(os.path.join(d, ALLOW), "a_test.py: nobody needs this\nb_test.py: needs a pack\n" + "c_test.py: needs the pinned llama.cpp checkout\n") + if check(d, quiet=True) == 0: + print(" FAIL an excuse for a registered test did not fail as stale", file=sys.stderr) + ok = False + else: + print(" ok an excuse that outlived its reason fails as stale") + # an excuse with no reason is refused rather than silently silencing a test + write(os.path.join(d, ALLOW), "b_test.py:\n") + try: + excuses(d) + print(" FAIL a reasonless excuse was accepted", file=sys.stderr) + ok = False + except SystemExit: + print(" ok a reasonless excuse is refused") + print("python_test_referenced_check self-test: %s" % ("ok" if ok else "FAILED")) + return 0 if ok else 1 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--check", action="store_true") + ap.add_argument("--self-test", action="store_true") + ap.add_argument("--root", default=os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + a = ap.parse_args() + if a.self_test: + return self_test() + if a.check: + return check(a.root) + print(__doc__.strip().splitlines()[-1], file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main())