Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions cmake/tests-pkg-proof.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# engine#2129: scripts/pkg-proof.sh carries a --self-test that no row ran. It exercises the row/summary accounting on stubs
# (the counter a packaging run prints is what tells a reader whether the run proved anything), and it needed no container,
# no device and no build -- there was simply no registration. tools/ci/tests_referenced_check.py now covers scripts/*.sh so
# the next one cannot be invisible either; this is the row for the one that was.
#
# A family file rather than a line in CMakeLists.txt: the glob at its end picks this up, so adding it touches no line that any
# other test-adding PR is editing.
add_test(NAME pkg_proof_self COMMAND ${CMAKE_SOURCE_DIR}/scripts/pkg-proof.sh --self-test)
39 changes: 33 additions & 6 deletions tools/ci/tests_referenced_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,20 @@ def referenced(root, need):


def unregistered_selftest_tools(root):
"""tools/**/*.py that declare a self-test and that no RUNNER names (engine#1901)."""
"""tools/**/*.py and scripts/*.sh that declare a self-test and that no RUNNER names (engine#1901, engine#2129).

`scripts/*.sh` was outside this corpus until a self-test there was found with no row at all: `scripts/pkg-proof.sh
--self-test` passes on its own and no ctest row or workflow step ran it, so the arm was dead code that every gate
called green. A self-test is a claim about a script, and the claim needs a runner whichever directory the script is in.
"""
tools = root / "tools"
if not tools.is_dir():
scripts = root / "scripts"
cand = []
if tools.is_dir():
cand += [p for p in tools.rglob("*.py")]
if scripts.is_dir():
cand += [p for p in scripts.glob("*.sh")]
if not cand:
return []
runner_text = []
for sub in RUNNERS:
Expand All @@ -142,7 +153,7 @@ def unregistered_selftest_tools(root):
except OSError:
pass
drift = []
for tool in sorted(tools.rglob("*.py")):
for tool in sorted(cand):
try:
src = tool.read_text(errors="replace")
except OSError:
Expand Down Expand Up @@ -245,8 +256,13 @@ def check(root):


def _selftest_tools(root):
tools = root / "tools"
return [p for p in tools.rglob("*.py") if SELFTEST.search(p.read_text(errors="replace"))] if tools.is_dir() else []
"""Every file the corpus above covers that declares a self-test -- counted with the same two directories."""
out = []
for sub, pat in (("tools", "**/*.py"), ("scripts", "*.sh")):
d = root / sub
if d.is_dir():
out += [p for p in d.glob(pat) if SELFTEST.search(p.read_text(errors="replace"))]
return out


def self_test():
Expand All @@ -263,9 +279,20 @@ def root_with(files):
p.write_text(body)
return d

cases_root = {} # hoisted: the scripts/ arms below need it, and it used to be created after case 1
# 0a/0b. a self-test in scripts/ -- the corpus that was missing until one was found with no row (engine#2129)
# (a clean tests/ corpus is part of the fixture: without one the tool answers 2 "cannot tell" before it reaches this)
base = {"tests/ok.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME ok COMMAND bash tests/ok.sh)\n"}
r = root_with(dict(base, **{"scripts/helper.sh": "#!/bin/sh\ncase \"${1:-}\" in --self-test) exit 0;; esac\n"}))
cases.append(("a scripts/*.sh self-test with no runner is drift", check(r), 1, "helper.sh"))
cases_root["a scripts/*.sh self-test with no runner is drift"] = r

r = root_with(dict(base, **{"scripts/helper.sh": "#!/bin/sh\ncase \"${1:-}\" in --self-test) exit 0;; esac\n",
"cmake/tests-helper.cmake": "add_test(NAME helper_self COMMAND bash scripts/helper.sh --self-test)\n"}))
cases.append(("...and a registration in cmake/*.cmake clears it", check(r), 0, None))

# 1. a test nobody mentions -> drift, and it is NAMED
r = root_with({"tests/lonely.sh": "#!/bin/sh\n", "CMakeLists.txt": "# nothing here\n"})
cases_root = {}
cases.append(("an unreferenced test is drift", check(r), 1, "lonely.sh")); cases_root["an unreferenced test is drift"] = r

# 2. referenced from CMakeLists.txt -> clean
Expand Down
Loading