Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,9 @@ add_python_test(pr_carry_check_self ${CMAKE_SOURCE_DIR}/tools/ci/pr_carry_check.
# engine#2021 / engine#2369 review: `registry_build.py --check-gaps` iterates the classes that are PRESENT, so a
# branch that deletes a reviewed class passes it -- measured on #2021, where 41 classes became 40 and --check-gaps
# still printed "none of the 41 reviewed significant classes is mapped". A test row for the three arms that do see it.
add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tests/registry_loss_check_test.py)
# engine#2445 review: run the tool, not the file, so CI exercises `--self-test` itself -- a row that
# invokes tests/<tool>_test.py directly never notices if the flag stops working. Same row names, same arms.
add_python_test(registry_loss_check_test ${CMAKE_SOURCE_DIR}/tools/ci/registry_loss_check.py --self-test)
# engine#2406 puts "audit those on a pin bump" on the f16 ratchet's green line; this is that audit
# for the HIP add-on's kernels, which the ratchet cannot see (it walks *.loom).
add_python_test(hip_f16_audit_self ${CMAKE_SOURCE_DIR}/tools/ci/hip_f16_audit.py --self-test)
Expand All @@ -460,10 +462,16 @@ add_test(NAME hip_f16_audit COMMAND ${Python3_EXECUTABLE}
set_tests_properties(hip_f16_audit PROPERTIES SKIP_RETURN_CODE 77)
# engine#2398: the stale-branch test, which `tests_referenced` refuses to leave unregistered -- a test no
# runner names is a test nobody runs.
add_python_test(branch_stale_check_test ${CMAKE_SOURCE_DIR}/tests/branch_stale_check_test.py)
# engine#2445 review: the rows run the tool, so CI exercises the flag (a row naming the test file never
# would). base's own addition here -- guard_loss_check_self -- is kept as it is.
add_python_test(branch_stale_check_test ${CMAKE_SOURCE_DIR}/tools/ci/branch_stale_check.py --self-test)
# engine#2429: the guard-loss check declares a self-test, so it needs a row -- tests_referenced_check caught
# this omission on the PR that added the tool, which is the check doing exactly its job.
add_python_test(guard_loss_check_self ${CMAKE_SOURCE_DIR}/tools/ci/guard_loss_check.py --self-test)
# engine#2445 review: and the flag itself must be load-bearing -- a row that ran the tool but
# dropped the child's status would stay green while both arms above went red. Measured with a
# scratch tree whose test file exits 1 (status dropped -> 0 vs 0), and with the flag ignored.
add_python_test(self_test_flag ${CMAKE_SOURCE_DIR}/tests/self_test_flag_test.py)

add_python_test(hyperloom_benchmark_test ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py)
# engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail.
Expand Down
98 changes: 98 additions & 0 deletions tests/self_test_flag_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""self_test_flag_test.py -- the `--self-test` flag must survive being ignored.

engine#2445 review: the flag's whole job is to *return the child's status*, so the failure
worth pinning is the quiet one. `subprocess.run([...])` without `.returncode` exits 0
whatever the arms said, and every row that calls the tool would stay green while the arms
went red. Equally, a flag that disappears reads as an argparse error (rc 2) rather than as
"no self-test".

Both are measured the same way: a copy of the tool in a scratch tree, pointed at a test
file that exits with a chosen status. Nothing here imports the tools, so the arms can move
without this file following them.

Exit: 0 the flag reports the child's status, 1 it does not.
"""
import os
import shutil
import subprocess
import sys
import tempfile

TOOLS = ("registry_loss_check", "branch_stale_check")


def tool_root(tmp, tool):
"""A scratch tree shaped like the repo: tools/ci/<tool>.py next to tests/."""
src = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tools", "ci", tool + ".py")
shutil.copyfile(src, os.path.join(tmp, "tools", "ci", tool + ".py"))
return os.path.join(tmp, "tools", "ci", tool + ".py")


def run_self_test(tmp, tool, child_rc):
"""Run the copied tool's --self-test against a test file that exits `child_rc`."""
test_path = os.path.join(tmp, "tests", tool + "_test.py")
with open(test_path, "w") as f:
f.write("import sys\nsys.exit(%d)\n" % child_rc)
return subprocess.run([sys.executable, os.path.join(tmp, "tools", "ci", tool + ".py"), "--self-test"],
capture_output=True, text=True)


def main():
fails = 0

def check(label, ok, detail=""):
nonlocal fails
if ok:
print("ok " + label)
else:
print("FAIL " + label + (" [" + detail + "]" if detail else ""))
fails += 1

for tool in TOOLS:
with tempfile.TemporaryDirectory() as tmp:
os.makedirs(os.path.join(tmp, "tools", "ci"))
os.makedirs(os.path.join(tmp, "tests"))
tool_root(tmp, tool)

# (1)+(2) THE ARMS THAT MATTER, and they are checked as a PAIR: the tool's status must
# equal the child's, so the two runs have to differ. Checking "child 1 -> rc != 0" alone
# is not enough -- a dropped `.returncode` falls through to the diff logic, which exits 1
# too, and that arm would pass for the wrong reason (measured: the first version of this
# file reported the fallout on the passing arm instead of naming the cause).
r_fail = run_self_test(tmp, tool, 1)
r_pass = run_self_test(tmp, tool, 0)
check("%s --self-test reports a passing child as 0 (rc %d, want 0)" % (tool, r_pass.returncode),
r_pass.returncode == 0, (r_pass.stderr or r_pass.stdout).strip()[-160:])
check("%s --self-test distinguishes a failing child (rc %d vs %d, want != and != 0)"
% (tool, r_fail.returncode, r_pass.returncode),
r_fail.returncode != 0 and r_fail.returncode != r_pass.returncode,
(r_fail.stderr or r_fail.stdout).strip()[-160:])

# (3) a removed flag is an argparse error, not a silently passing row. This is what
# the old rows could not see: they invoked the test file and never touched the flag.
r = subprocess.run([sys.executable, os.path.join(tmp, "tools", "ci", tool + ".py"), "--nonsense"],
capture_output=True, text=True)
check("%s rejects an unknown flag with rc 2 rather than 0 (rc %d)" % (tool, r.returncode),
r.returncode == 2, (r.stderr or "").strip()[-160:])

print("self_test_flag: %s" % ("ok" if fails == 0 else "%d failure(s)" % fails))
return 0 if fails == 0 else 1


if __name__ == "__main__":
sys.exit(main())
10 changes: 10 additions & 0 deletions tools/ci/branch_stale_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
"""

import argparse
import os
import subprocess
import sys

Expand Down Expand Up @@ -72,9 +73,18 @@ def main():
parser = argparse.ArgumentParser(description="Report what merging the branch would lose from the base.")
parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)")
parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)")
# engine#2443: the other two tools in tools/ci carry --self-test and these two kept their arms in
# tests/*_test.py, so `--self-test` here answered "unrecognized arguments" -- which reads as "there is
# no self-test" rather than "its arms live elsewhere". The arms stay where they are; this runs them.
parser.add_argument("--self-test", action="store_true",
help="run the arms in tests/branch_stale_check_test.py and exit with their status")
parser.add_argument("--quiet", action="store_true", help="only print failures")
parser.add_argument("--limit", type=int, default=25, help="how many files to name (default: 25)")
args = parser.parse_args()
if args.self_test:
return subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))), "tests/branch_stale_check_test.py")]).returncode


for label, rev in (("base", args.base), ("head", args.head)):
if not ref_exists(rev):
Expand Down
10 changes: 10 additions & 0 deletions tools/ci/registry_loss_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
"""

import argparse
import os
import json
import subprocess
import sys
Expand Down Expand Up @@ -157,8 +158,17 @@ def main():
parser = argparse.ArgumentParser(description="Refuse a branch that drops registry rows the base has.")
parser.add_argument("--base", default="origin/main", help="base ref (default: origin/main)")
parser.add_argument("--head", default="HEAD", help="head ref (default: HEAD)")
# engine#2443: the other two tools in tools/ci carry --self-test and these two kept their arms in
# tests/*_test.py, so `--self-test` here answered "unrecognized arguments" -- which reads as "there is
# no self-test" rather than "its arms live elsewhere". The arms stay where they are; this runs them.
parser.add_argument("--self-test", action="store_true",
help="run the arms in tests/registry_loss_check_test.py and exit with their status")
parser.add_argument("--quiet", action="store_true", help="only print failures")
args = parser.parse_args()
if args.self_test:
return subprocess.run([sys.executable, os.path.join(os.path.dirname(os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))), "tests/registry_loss_check_test.py")]).returncode


problems = []
for label, rev in (("base", args.base), ("head", args.head)):
Expand Down
22 changes: 21 additions & 1 deletion tools/ci/tests_referenced_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,9 +241,16 @@ def unregistered_py_tests(root):
runner_text.append(path.read_text(errors="replace"))
except OSError:
continue
# engine#2445: a row that runs `tools/ci/<tool>.py --self-test` DOES run tests/<tool>_test.py -- that
# is what the flag executes. Without this, rewiring a row from the file to the tool (so CI exercises
# the flag at all) reads as "no runner names the test", and the check fails on the improvement.
selftest_named = set()
for t in runner_text:
for m in re.finditer(r"tools/ci/([A-Za-z0-9_]+)\.py\s+--self[-_]?test", t):
selftest_named.add(m.group(1) + "_test.py")
out = []
for q in cand:
if q.name in imported:
if q.name in imported or q.name in selftest_named:
continue
if not any(q.name in t for t in runner_text):
out.append(str(q.relative_to(root)))
Expand Down Expand Up @@ -331,6 +338,19 @@ def root_with(files):
cases.append(("...and a registration in cmake/*.cmake clears it", check(r), 0, None))

# 1. a test nobody mentions -> drift, and it is NAMED
# 1a-ii. engine#2445: a row may run the TOOL's --self-test instead of the test file, because that is
# what makes CI exercise the flag. Both halves are pinned: with the flag it counts as a row, and the
# same row without `--self-test` does not (otherwise this rule would excuse a row that runs nothing).
r = root_with({"tests/foo_test.py": "print('ok')\n",
"tools/ci/foo.py": "import argparse\nargparse.ArgumentParser().add_argument('--self-test', action='store_true')\n",
"CMakeLists.txt": "add_python_test(foo ${CMAKE_SOURCE_DIR}/tools/ci/foo.py --self-test)\n"})
cases.append(("a row running tools/ci/foo.py --self-test references tests/foo_test.py", check(r), 0, None))
r = root_with({"tests/foo_test.py": "print('ok')\n",
"tools/ci/foo.py": "print('no flag here')\n",
"CMakeLists.txt": "add_python_test(foo ${CMAKE_SOURCE_DIR}/tools/ci/foo.py --check)\n"})
cases.append(("...but the same row WITHOUT --self-test does not", check(r), 1, "foo_test.py"))
cases_root["...but the same row WITHOUT --self-test does not"] = r

r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"})
cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r

Expand Down
Loading