From ecdd62438d243708dcf00edfababb6a944ca69d5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 21:22:09 -0400 Subject: [PATCH] tests_referenced: cover scripts/*.sh self-tests, which had no coverage and one dead arm (engine#2129) `tools/ci/tests_referenced_check.py` covers `tests/*.sh`, `tests/*_test.py` and `tools/**/*.py` -- self-tests in `scripts/` were outside its corpus. So this was invisible: scripts/pkg-proof.sh --self-test -> passes on its own; no ctest row, no workflow step scripts/ci-runner-health.sh --self-test -> registered (2 mentions) scripts/npu-window11.sh --self-test -> registered (2 mentions) One of three was dead code that every gate called green, which is the class this tool exists for and the third time today a check was green because it was not looking. `pkg-proof.sh --self-test` exercises the row/summary accounting on stubs (it is what tells a reader whether a packaging run proved anything), and it needs no container, no device and no build -- there was simply no row. Two changes, and the second is the one that matters: * the corpus now includes `scripts/*.sh`, counted by the same two-directory walk as the tool self-tests, so the summary line cannot claim more than the corpus covers (34 -> 38 self-tests, the three scripts plus one more tool); * `cmake/tests-pkg-proof.cmake` registers the arm that was missing. A family file rather than a line in CMakeLists.txt, because the glob at its end picks it up and 25 open PRs are editing that file. Verified both directions on the real tree: with the row -> "all 85 test script(s) are referenced ...; 38 tool/script self-test(s) ...", rc 0 without the row -> "scripts/pkg-proof.sh declares a self-test and no runner names it -- ... the arm is dead code", rc 1 Two arms in the tool's own self-test pin the new corpus, and the fixture carries a clean `tests/` corpus deliberately: with an empty one the tool answers 2 "cannot tell" before it ever reaches the scripts/ file, which is how my first version of the arm failed (rc 2, wanted 1). --- cmake/tests-pkg-proof.cmake | 23 ++++++++++++++++++ tools/ci/tests_referenced_check.py | 39 +++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 cmake/tests-pkg-proof.cmake diff --git a/cmake/tests-pkg-proof.cmake b/cmake/tests-pkg-proof.cmake new file mode 100644 index 000000000..a1b54e0e6 --- /dev/null +++ b/cmake/tests-pkg-proof.cmake @@ -0,0 +1,23 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#2129: scripts/pkg-proof.sh carries a --self-test that no row ran. It exercises the row/summary accounting on stubs +# (the counter a packaging run prints is what tells a reader whether the run proved anything), and it needed no container, +# no device and no build -- there was simply no registration. tools/ci/tests_referenced_check.py now covers scripts/*.sh so +# the next one cannot be invisible either; this is the row for the one that was. +# +# A family file rather than a line in CMakeLists.txt: the glob at its end picks this up, so adding it touches no line that any +# other test-adding PR is editing. +add_test(NAME pkg_proof_self COMMAND ${CMAKE_SOURCE_DIR}/scripts/pkg-proof.sh --self-test) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 8dd2f3818..e0ed420ff 100644 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -128,9 +128,20 @@ def referenced(root, need): def unregistered_selftest_tools(root): - """tools/**/*.py that declare a self-test and that no RUNNER names (engine#1901).""" + """tools/**/*.py and scripts/*.sh that declare a self-test and that no RUNNER names (engine#1901, engine#2129). + + `scripts/*.sh` was outside this corpus until a self-test there was found with no row at all: `scripts/pkg-proof.sh + --self-test` passes on its own and no ctest row or workflow step ran it, so the arm was dead code that every gate + called green. A self-test is a claim about a script, and the claim needs a runner whichever directory the script is in. + """ tools = root / "tools" - if not tools.is_dir(): + scripts = root / "scripts" + cand = [] + if tools.is_dir(): + cand += [p for p in tools.rglob("*.py")] + if scripts.is_dir(): + cand += [p for p in scripts.glob("*.sh")] + if not cand: return [] runner_text = [] for sub in RUNNERS: @@ -142,7 +153,7 @@ def unregistered_selftest_tools(root): except OSError: pass drift = [] - for tool in sorted(tools.rglob("*.py")): + for tool in sorted(cand): try: src = tool.read_text(errors="replace") except OSError: @@ -245,8 +256,13 @@ def check(root): def _selftest_tools(root): - tools = root / "tools" - return [p for p in tools.rglob("*.py") if SELFTEST.search(p.read_text(errors="replace"))] if tools.is_dir() else [] + """Every file the corpus above covers that declares a self-test -- counted with the same two directories.""" + out = [] + for sub, pat in (("tools", "**/*.py"), ("scripts", "*.sh")): + d = root / sub + if d.is_dir(): + out += [p for p in d.glob(pat) if SELFTEST.search(p.read_text(errors="replace"))] + return out def self_test(): @@ -263,9 +279,20 @@ def root_with(files): p.write_text(body) return d + cases_root = {} # hoisted: the scripts/ arms below need it, and it used to be created after case 1 + # 0a/0b. a self-test in scripts/ -- the corpus that was missing until one was found with no row (engine#2129) + # (a clean tests/ corpus is part of the fixture: without one the tool answers 2 "cannot tell" before it reaches this) + base = {"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n"} + r = root_with(dict(base, **{"scripts/helper.sh": "#!/bin/sh\ncase \"${1:-}\" in --self-test) exit 0;; esac\n"})) + cases.append(("a scripts/*.sh self-test with no runner is drift", check(r), 1, "helper.sh")) + cases_root["a scripts/*.sh self-test with no runner is drift"] = r + + r = root_with(dict(base, **{"scripts/helper.sh": "#!/bin/sh\ncase \"${1:-}\" in --self-test) exit 0;; esac\n", + "cmake/tests-helper.cmake": "add_test(NAME helper_self COMMAND bash scripts/helper.sh --self-test)\n"})) + cases.append(("...and a registration in cmake/*.cmake clears it", check(r), 0, None)) + # 1. a test nobody mentions -> drift, and it is NAMED r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"}) - cases_root = {} cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r # 2. referenced from CMakeLists.txt -> clean