diff --git a/CMakeLists.txt b/CMakeLists.txt index fb5eae707..1a5255a9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -448,7 +448,9 @@ add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check. # engine#2021 / engine#2369 review: `registry_build.py --check-gaps` iterates the classes that are PRESENT, so a # branch that deletes a reviewed class passes it -- measured on #2021, where 41 classes became 40 and --check-gaps # still printed "none of the 41 reviewed significant classes is mapped". A test row for the three arms that do see it. -add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tests/registry_loss_check_test.py) +# engine#2445 review: run the tool, not the file, so CI exercises `--self-test` itself -- a row that +# invokes tests/_test.py directly never notices if the flag stops working. Same row names, same arms. +add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tools/ci/registry_loss_check.py --self-test) # engine#2406 puts "audit those on a pin bump" on the f16 ratchet's green line; this is that audit # for the HIP add-on's kernels, which the ratchet cannot see (it walks *.loom). add_python_test(hip_f16_audit_self ${CMAKE_SOURCE_DIR}/tools/ci/hip_f16_audit.py --self-test) @@ -460,10 +462,16 @@ add_test(NAME hip_f16_audit COMMAND ${Python3_EXECUTABLE} set_tests_properties(hip_f16_audit PROPERTIES SKIP_RETURN_CODE 77) # engine#2398: the stale-branch test, which `tests_referenced` refuses to leave unregistered -- a test no # runner names is a test nobody runs. -add_python_test(branch_stale_check_test ${CMAKE_SOURCE_DIR}/tests/branch_stale_check_test.py) +# engine#2445 review: the rows run the tool, so CI exercises the flag (a row naming the test file never +# would). base's own addition here -- guard_loss_check_self -- is kept as it is. +add_python_test(branch_stale_check_test ${CMAKE_SOURCE_DIR}/tools/ci/branch_stale_check.py --self-test) # engine#2429: the guard-loss check declares a self-test, so it needs a row -- tests_referenced_check caught # this omission on the PR that added the tool, which is the check doing exactly its job. add_python_test(guard_loss_check_self ${CMAKE_SOURCE_DIR}/tools/ci/guard_loss_check.py --self-test) +# engine#2445 review: and the flag itself must be load-bearing -- a row that ran the tool but +# dropped the child's status would stay green while both arms above went red. Measured with a +# scratch tree whose test file exits 1 (status dropped -> 0 vs 0), and with the flag ignored. +add_python_test(self_test_flag ${CMAKE_SOURCE_DIR}/tests/self_test_flag_test.py) add_python_test(hyperloom_benchmark_test ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py) # engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail. diff --git a/tests/self_test_flag_test.py b/tests/self_test_flag_test.py new file mode 100644 index 000000000..68249292a --- /dev/null +++ b/tests/self_test_flag_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""self_test_flag_test.py -- the `--self-test` flag must survive being ignored. + +engine#2445 review: the flag's whole job is to *return the child's status*, so the failure +worth pinning is the quiet one. `subprocess.run([...])` without `.returncode` exits 0 +whatever the arms said, and every row that calls the tool would stay green while the arms +went red. Equally, a flag that disappears reads as an argparse error (rc 2) rather than as +"no self-test". + +Both are measured the same way: a copy of the tool in a scratch tree, pointed at a test +file that exits with a chosen status. Nothing here imports the tools, so the arms can move +without this file following them. + +Exit: 0 the flag reports the child's status, 1 it does not. +""" +import os +import shutil +import subprocess +import sys +import tempfile + +TOOLS = ("registry_loss_check", "branch_stale_check") + + +def tool_root(tmp, tool): + """A scratch tree shaped like the repo: tools/ci/.py next to tests/.""" + src = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools", "ci", tool + ".py") + shutil.copyfile(src, os.path.join(tmp, "tools", "ci", tool + ".py")) + return os.path.join(tmp, "tools", "ci", tool + ".py") + + +def run_self_test(tmp, tool, child_rc): + """Run the copied tool's --self-test against a test file that exits `child_rc`.""" + test_path = os.path.join(tmp, "tests", tool + "_test.py") + with open(test_path, "w") as f: + f.write("import sys\nsys.exit(%d)\n" % child_rc) + return subprocess.run([sys.executable, os.path.join(tmp, "tools", "ci", tool + ".py"), "--self-test"], + capture_output=True, text=True) + + +def main(): + fails = 0 + + def check(label, ok, detail=""): + nonlocal fails + if ok: + print("ok " + label) + else: + print("FAIL " + label + (" [" + detail + "]" if detail else "")) + fails += 1 + + for tool in TOOLS: + with tempfile.TemporaryDirectory() as tmp: + os.makedirs(os.path.join(tmp, "tools", "ci")) + os.makedirs(os.path.join(tmp, "tests")) + tool_root(tmp, tool) + + # (1)+(2) THE ARMS THAT MATTER, and they are checked as a PAIR: the tool's status must + # equal the child's, so the two runs have to differ. Checking "child 1 -> rc != 0" alone + # is not enough -- a dropped `.returncode` falls through to the diff logic, which exits 1 + # too, and that arm would pass for the wrong reason (measured: the first version of this + # file reported the fallout on the passing arm instead of naming the cause). + r_fail = run_self_test(tmp, tool, 1) + r_pass = run_self_test(tmp, tool, 0) + check("%s --self-test reports a passing child as 0 (rc %d, want 0)" % (tool, r_pass.returncode), + r_pass.returncode == 0, (r_pass.stderr or r_pass.stdout).strip()[-160:]) + check("%s --self-test distinguishes a failing child (rc %d vs %d, want != and != 0)" + % (tool, r_fail.returncode, r_pass.returncode), + r_fail.returncode != 0 and r_fail.returncode != r_pass.returncode, + (r_fail.stderr or r_fail.stdout).strip()[-160:]) + + # (3) a removed flag is an argparse error, not a silently passing row. This is what + # the old rows could not see: they invoked the test file and never touched the flag. + r = subprocess.run([sys.executable, os.path.join(tmp, "tools", "ci", tool + ".py"), "--nonsense"], + capture_output=True, text=True) + check("%s rejects an unknown flag with rc 2 rather than 0 (rc %d)" % (tool, r.returncode), + r.returncode == 2, (r.stderr or "").strip()[-160:]) + + print("self_test_flag: %s" % ("ok" if fails == 0 else "%d failure(s)" % fails)) + return 0 if fails == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci/branch_stale_check.py b/tools/ci/branch_stale_check.py index 49130d2bc..14e75574c 100755 --- a/tools/ci/branch_stale_check.py +++ b/tools/ci/branch_stale_check.py @@ -35,6 +35,7 @@ """ import argparse +import os import subprocess import sys @@ -72,9 +73,18 @@ def main(): parser = argparse.ArgumentParser(description="Report what merging the branch would lose from the base.") parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)") parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)") + # engine#2443: the other two tools in tools/ci carry --self-test and these two kept their arms in + # tests/*_test.py, so `--self-test` here answered "unrecognized arguments" -- which reads as "there is + # no self-test" rather than "its arms live elsewhere". The arms stay where they are; this runs them. + parser.add_argument("--self-test", action="store_true", + help="run the arms in tests/branch_stale_check_test.py and exit with their status") parser.add_argument("--quiet", action="store_true", help="only print failures") parser.add_argument("--limit", type=int, default=25, help="how many files to name (default: 25)") args = parser.parse_args() + if args.self_test: + return subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))), "tests/branch_stale_check_test.py")]).returncode + for label, rev in (("base", args.base), ("head", args.head)): if not ref_exists(rev): diff --git a/tools/ci/registry_loss_check.py b/tools/ci/registry_loss_check.py index 8696ef421..246a916d6 100644 --- a/tools/ci/registry_loss_check.py +++ b/tools/ci/registry_loss_check.py @@ -35,6 +35,7 @@ """ import argparse +import os import json import subprocess import sys @@ -157,8 +158,17 @@ def main(): parser = argparse.ArgumentParser(description="Refuse a branch that drops registry rows the base has.") parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)") parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)") + # engine#2443: the other two tools in tools/ci carry --self-test and these two kept their arms in + # tests/*_test.py, so `--self-test` here answered "unrecognized arguments" -- which reads as "there is + # no self-test" rather than "its arms live elsewhere". The arms stay where they are; this runs them. + parser.add_argument("--self-test", action="store_true", + help="run the arms in tests/registry_loss_check_test.py and exit with their status") parser.add_argument("--quiet", action="store_true", help="only print failures") args = parser.parse_args() + if args.self_test: + return subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__)))), "tests/registry_loss_check_test.py")]).returncode + problems = [] for label, rev in (("base", args.base), ("head", args.head)): diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index e2b373bd8..f54093cc1 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -241,9 +241,16 @@ def unregistered_py_tests(root): runner_text.append(path.read_text(errors="replace")) except OSError: continue + # engine#2445: a row that runs `tools/ci/.py --self-test` DOES run tests/_test.py -- that + # is what the flag executes. Without this, rewiring a row from the file to the tool (so CI exercises + # the flag at all) reads as "no runner names the test", and the check fails on the improvement. + selftest_named = set() + for t in runner_text: + for m in re.finditer(r"tools/ci/([A-Za-z0-9_]+)\.py\s+--self[-_]?test", t): + selftest_named.add(m.group(1) + "_test.py") out = [] for q in cand: - if q.name in imported: + if q.name in imported or q.name in selftest_named: continue if not any(q.name in t for t in runner_text): out.append(str(q.relative_to(root))) @@ -331,6 +338,19 @@ def root_with(files): cases.append(("...and a registration in cmake/*.cmake clears it", check(r), 0, None)) # 1. a test nobody mentions -> drift, and it is NAMED + # 1a-ii. engine#2445: a row may run the TOOL's --self-test instead of the test file, because that is + # what makes CI exercise the flag. Both halves are pinned: with the flag it counts as a row, and the + # same row without `--self-test` does not (otherwise this rule would excuse a row that runs nothing). + r = root_with({"tests/foo_test.py": "print('ok')\n", + "tools/ci/foo.py": "import argparse\nargparse.ArgumentParser().add_argument('--self-test', action='store_true')\n", + "CMakeLists.txt": "add_python_test(foo ${CMAKE_SOURCE_DIR}/tools/ci/foo.py --self-test)\n"}) + cases.append(("a row running tools/ci/foo.py --self-test references tests/foo_test.py", check(r), 0, None)) + r = root_with({"tests/foo_test.py": "print('ok')\n", + "tools/ci/foo.py": "print('no flag here')\n", + "CMakeLists.txt": "add_python_test(foo ${CMAKE_SOURCE_DIR}/tools/ci/foo.py --check)\n"}) + cases.append(("...but the same row WITHOUT --self-test does not", check(r), 1, "foo_test.py")) + cases_root["...but the same row WITHOUT --self-test does not"] = r + r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"}) cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r