From 82793ed24ff55ad87066e1044c4b8130c411d91b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 14:00:17 -0400 Subject: [PATCH 1/2] tools/ci: the two checks keep their arms in tests/, so give them --self-test too Four tools in tools/ci carry a self-test; two of them (hip_f16_audit, pr_review_queue) answer `--self-test` and two (registry_loss_check, branch_stale_check) kept their arms in tests/*_test.py and answered 'unrecognized arguments: --self-test'. That reads as 'there is no self-test' rather than 'its arms live elsewhere' -- I hit it myself while sweeping my own tools after a merge, and spent a minute concluding the check had no arms before finding them in tests/. The arms stay where they are -- one file per tool, already registered as ctest rows -- and --self-test now runs exactly that file through the same interpreter and returns its status. Nothing is duplicated and no arm moved. Verified: both answer `--self-test` rc 0 ('OK' from unittest), the normal modes are unchanged (registry_loss / branch_stale on main vs main -> rc 0), and the flag really reports a failing arm: mutating registry_loss_check's merge-tree classification makes `--self-test` exit 1, and restoring it exits 0. --- tools/ci/branch_stale_check.py | 10 ++++++++++ tools/ci/registry_loss_check.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/tools/ci/branch_stale_check.py b/tools/ci/branch_stale_check.py index 49130d2bc..14e75574c 100755 --- a/tools/ci/branch_stale_check.py +++ b/tools/ci/branch_stale_check.py @@ -35,6 +35,7 @@ """ import argparse +import os import subprocess import sys @@ -72,9 +73,18 @@ def main(): parser = argparse.ArgumentParser(description="Report what merging the branch would lose from the base.") parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)") parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)") + # engine#2443: the other two tools in tools/ci carry --self-test and these two kept their arms in + # tests/*_test.py, so `--self-test` here answered "unrecognized arguments" -- which reads as "there is + # no self-test" rather than "its arms live elsewhere". The arms stay where they are; this runs them. + parser.add_argument("--self-test", action="store_true", + help="run the arms in tests/branch_stale_check_test.py and exit with their status") parser.add_argument("--quiet", action="store_true", help="only print failures") parser.add_argument("--limit", type=int, default=25, help="how many files to name (default: 25)") args = parser.parse_args() + if args.self_test: + return subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))), "tests/branch_stale_check_test.py")]).returncode + for label, rev in (("base", args.base), ("head", args.head)): if not ref_exists(rev): diff --git a/tools/ci/registry_loss_check.py b/tools/ci/registry_loss_check.py index 8696ef421..246a916d6 100644 --- a/tools/ci/registry_loss_check.py +++ b/tools/ci/registry_loss_check.py @@ -35,6 +35,7 @@ """ import argparse +import os import json import subprocess import sys @@ -157,8 +158,17 @@ def main(): parser = argparse.ArgumentParser(description="Refuse a branch that drops registry rows the base has.") parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)") parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)") + # engine#2443: the other two tools in tools/ci carry --self-test and these two kept their arms in + # tests/*_test.py, so `--self-test` here answered "unrecognized arguments" -- which reads as "there is + # no self-test" rather than "its arms live elsewhere". The arms stay where they are; this runs them. + parser.add_argument("--self-test", action="store_true", + help="run the arms in tests/registry_loss_check_test.py and exit with their status") parser.add_argument("--quiet", action="store_true", help="only print failures") args = parser.parse_args() + if args.self_test: + return subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))), "tests/registry_loss_check_test.py")]).returncode + problems = [] for label, rev in (("base", args.base), ("head", args.head)): From 5ee5cdd2d8dbe8013193e3c99bedc8266122c6f7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 19:35:41 -0400 Subject: [PATCH 2/2] tools/ci: make the --self-test flag load-bearing, and run it from the rows that name the arms The review on #2445 is right: the flag I added had nothing that would notice if it stopped working. Two changes, and the second is what makes the first checkable. 1. The two CI rows now run the TOOL with --self-test instead of invoking the test file directly. Same row names, same arms, same place -- but CI exercises the flag, so 'unrecognized arguments: --self-test' (the sentence this PR exists to remove) can no longer reappear unseen. hip_f16_audit_self and tests_referenced_self already read this way. 2. tests/self_test_flag_test.py pins the flag's behaviour against a tool copy pointed at a test file of a chosen status: passing child -> 0, failing child -> non-zero AND different from the passing run. The pair matters, because dropping '.returncode' falls through to the diff logic, which exits 1 as well -- an arm that only checked 'failing child -> rc 1' would pass for the wrong reason (measured: the first version of this file reported the fallout on the passing arm instead of naming the cause). Rewiring the rows made tests_referenced_check fail on its own tree -- both tests became 'tests that no runner names' -- which is the check doing its job. It now recognises that a row naming 'tools/ci/.py --self-test' runs tests/_test.py, which is exactly what the flag executes; two arms pin it (the row with the flag counts, the same row without it does not). Verified: self_test_flag ok; both mutations caught with accurate messages (dropped status -> 'rc 1 vs 1, want != and != 0'; flag ignored -> 'rc 0 vs 0'); tests_referenced_check --check rc 0 and its --self-test 0 failures; pre-pr-checks all 9 green, 1 skipped; copyright rc 0. --- CMakeLists.txt | 12 +++- tests/self_test_flag_test.py | 98 ++++++++++++++++++++++++++++++ tools/ci/tests_referenced_check.py | 22 ++++++- 3 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 tests/self_test_flag_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index fb5eae707..1a5255a9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -448,7 +448,9 @@ add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check. # engine#2021 / engine#2369 review: `registry_build.py --check-gaps` iterates the classes that are PRESENT, so a # branch that deletes a reviewed class passes it -- measured on #2021, where 41 classes became 40 and --check-gaps # still printed "none of the 41 reviewed significant classes is mapped". A test row for the three arms that do see it. -add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tests/registry_loss_check_test.py) +# engine#2445 review: run the tool, not the file, so CI exercises `--self-test` itself -- a row that +# invokes tests/_test.py directly never notices if the flag stops working. Same row names, same arms. +add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tools/ci/registry_loss_check.py --self-test) # engine#2406 puts "audit those on a pin bump" on the f16 ratchet's green line; this is that audit # for the HIP add-on's kernels, which the ratchet cannot see (it walks *.loom). add_python_test(hip_f16_audit_self ${CMAKE_SOURCE_DIR}/tools/ci/hip_f16_audit.py --self-test) @@ -460,10 +462,16 @@ add_test(NAME hip_f16_audit COMMAND ${Python3_EXECUTABLE} set_tests_properties(hip_f16_audit PROPERTIES SKIP_RETURN_CODE 77) # engine#2398: the stale-branch test, which `tests_referenced` refuses to leave unregistered -- a test no # runner names is a test nobody runs. -add_python_test(branch_stale_check_test ${CMAKE_SOURCE_DIR}/tests/branch_stale_check_test.py) +# engine#2445 review: the rows run the tool, so CI exercises the flag (a row naming the test file never +# would). base's own addition here -- guard_loss_check_self -- is kept as it is. +add_python_test(branch_stale_check_test ${CMAKE_SOURCE_DIR}/tools/ci/branch_stale_check.py --self-test) # engine#2429: the guard-loss check declares a self-test, so it needs a row -- tests_referenced_check caught # this omission on the PR that added the tool, which is the check doing exactly its job. add_python_test(guard_loss_check_self ${CMAKE_SOURCE_DIR}/tools/ci/guard_loss_check.py --self-test) +# engine#2445 review: and the flag itself must be load-bearing -- a row that ran the tool but +# dropped the child's status would stay green while both arms above went red. Measured with a +# scratch tree whose test file exits 1 (status dropped -> 0 vs 0), and with the flag ignored. +add_python_test(self_test_flag ${CMAKE_SOURCE_DIR}/tests/self_test_flag_test.py) add_python_test(hyperloom_benchmark_test ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py) # engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail. diff --git a/tests/self_test_flag_test.py b/tests/self_test_flag_test.py new file mode 100644 index 000000000..68249292a --- /dev/null +++ b/tests/self_test_flag_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""self_test_flag_test.py -- the `--self-test` flag must survive being ignored. + +engine#2445 review: the flag's whole job is to *return the child's status*, so the failure +worth pinning is the quiet one. `subprocess.run([...])` without `.returncode` exits 0 +whatever the arms said, and every row that calls the tool would stay green while the arms +went red. Equally, a flag that disappears reads as an argparse error (rc 2) rather than as +"no self-test". + +Both are measured the same way: a copy of the tool in a scratch tree, pointed at a test +file that exits with a chosen status. Nothing here imports the tools, so the arms can move +without this file following them. + +Exit: 0 the flag reports the child's status, 1 it does not. +""" +import os +import shutil +import subprocess +import sys +import tempfile + +TOOLS = ("registry_loss_check", "branch_stale_check") + + +def tool_root(tmp, tool): + """A scratch tree shaped like the repo: tools/ci/.py next to tests/.""" + src = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools", "ci", tool + ".py") + shutil.copyfile(src, os.path.join(tmp, "tools", "ci", tool + ".py")) + return os.path.join(tmp, "tools", "ci", tool + ".py") + + +def run_self_test(tmp, tool, child_rc): + """Run the copied tool's --self-test against a test file that exits `child_rc`.""" + test_path = os.path.join(tmp, "tests", tool + "_test.py") + with open(test_path, "w") as f: + f.write("import sys\nsys.exit(%d)\n" % child_rc) + return subprocess.run([sys.executable, os.path.join(tmp, "tools", "ci", tool + ".py"), "--self-test"], + capture_output=True, text=True) + + +def main(): + fails = 0 + + def check(label, ok, detail=""): + nonlocal fails + if ok: + print("ok " + label) + else: + print("FAIL " + label + (" [" + detail + "]" if detail else "")) + fails += 1 + + for tool in TOOLS: + with tempfile.TemporaryDirectory() as tmp: + os.makedirs(os.path.join(tmp, "tools", "ci")) + os.makedirs(os.path.join(tmp, "tests")) + tool_root(tmp, tool) + + # (1)+(2) THE ARMS THAT MATTER, and they are checked as a PAIR: the tool's status must + # equal the child's, so the two runs have to differ. Checking "child 1 -> rc != 0" alone + # is not enough -- a dropped `.returncode` falls through to the diff logic, which exits 1 + # too, and that arm would pass for the wrong reason (measured: the first version of this + # file reported the fallout on the passing arm instead of naming the cause). + r_fail = run_self_test(tmp, tool, 1) + r_pass = run_self_test(tmp, tool, 0) + check("%s --self-test reports a passing child as 0 (rc %d, want 0)" % (tool, r_pass.returncode), + r_pass.returncode == 0, (r_pass.stderr or r_pass.stdout).strip()[-160:]) + check("%s --self-test distinguishes a failing child (rc %d vs %d, want != and != 0)" + % (tool, r_fail.returncode, r_pass.returncode), + r_fail.returncode != 0 and r_fail.returncode != r_pass.returncode, + (r_fail.stderr or r_fail.stdout).strip()[-160:]) + + # (3) a removed flag is an argparse error, not a silently passing row. This is what + # the old rows could not see: they invoked the test file and never touched the flag. + r = subprocess.run([sys.executable, os.path.join(tmp, "tools", "ci", tool + ".py"), "--nonsense"], + capture_output=True, text=True) + check("%s rejects an unknown flag with rc 2 rather than 0 (rc %d)" % (tool, r.returncode), + r.returncode == 2, (r.stderr or "").strip()[-160:]) + + print("self_test_flag: %s" % ("ok" if fails == 0 else "%d failure(s)" % fails)) + return 0 if fails == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index e2b373bd8..f54093cc1 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -241,9 +241,16 @@ def unregistered_py_tests(root): runner_text.append(path.read_text(errors="replace")) except OSError: continue + # engine#2445: a row that runs `tools/ci/.py --self-test` DOES run tests/_test.py -- that + # is what the flag executes. Without this, rewiring a row from the file to the tool (so CI exercises + # the flag at all) reads as "no runner names the test", and the check fails on the improvement. + selftest_named = set() + for t in runner_text: + for m in re.finditer(r"tools/ci/([A-Za-z0-9_]+)\.py\s+--self[-_]?test", t): + selftest_named.add(m.group(1) + "_test.py") out = [] for q in cand: - if q.name in imported: + if q.name in imported or q.name in selftest_named: continue if not any(q.name in t for t in runner_text): out.append(str(q.relative_to(root))) @@ -331,6 +338,19 @@ def root_with(files): cases.append(("...and a registration in cmake/*.cmake clears it", check(r), 0, None)) # 1. a test nobody mentions -> drift, and it is NAMED + # 1a-ii. engine#2445: a row may run the TOOL's --self-test instead of the test file, because that is + # what makes CI exercise the flag. Both halves are pinned: with the flag it counts as a row, and the + # same row without `--self-test` does not (otherwise this rule would excuse a row that runs nothing). + r = root_with({"tests/foo_test.py": "print('ok')\n", + "tools/ci/foo.py": "import argparse\nargparse.ArgumentParser().add_argument('--self-test', action='store_true')\n", + "CMakeLists.txt": "add_python_test(foo ${CMAKE_SOURCE_DIR}/tools/ci/foo.py --self-test)\n"}) + cases.append(("a row running tools/ci/foo.py --self-test references tests/foo_test.py", check(r), 0, None)) + r = root_with({"tests/foo_test.py": "print('ok')\n", + "tools/ci/foo.py": "print('no flag here')\n", + "CMakeLists.txt": "add_python_test(foo ${CMAKE_SOURCE_DIR}/tools/ci/foo.py --check)\n"}) + cases.append(("...but the same row WITHOUT --self-test does not", check(r), 1, "foo_test.py")) + cases_root["...but the same row WITHOUT --self-test does not"] = r + r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"}) cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r