From 3aab4e410601017c35d13df9980d14b78e58554d Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 10 Nov 2026 03:11:45 -0400 Subject: [PATCH] ci: the unreferenced-tests allow-list actually enforces both directions (engine#1820) Measured on merged main: the row's second direction did not bite. - an allow entry for a test that is gone printed its warning and the row still exited 0; - an allow entry for a test that IS referenced was accepted in silence and counted as 'deliberately not a row', so the list could rot exactly as the file's own docstring says it must not; - a path-form entry (tests/foo.sh, the form the issue's hand-loop printed and the form tools/ci/ otherwise uses) matched no test, so it suppressed nothing and was then reported as 'no such test exists any more' -- two contradicting messages about one line, the format never named. Now: gone and idle entries both fail with their own message, entries are normalised to basenames so either form works, and the self-test gains an arm for each (plus the path form). Its old arm 7 asserted 'reported, but not a failure', which is the defect written down as intent. Green on main's tree: all 73 test scripts referenced (1 deliberately not a row). --- tools/ci/tests_referenced_check.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 93df3264f..90ba398f8 100755 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -60,7 +60,10 @@ def allow_list(root): if not line or line.startswith("#"): continue name, _, reason = line.partition("#") - out[name.strip()] = reason.strip() + # Normalised to a basename: `tests/foo.sh` and `foo.sh` mean the same entry. Measured before this: a + # path-form entry matched no test, so it suppressed nothing AND was reported as "no such test exists any + # more", two contradicting messages about one line and the format never named. + out[pathlib.PurePosixPath(name.strip().rstrip("/")).name] = reason.strip() return out @@ -104,13 +107,17 @@ def check(root): allow = allow_list(root) found = referenced(root, need) drift = [n for n in need if not found[n] and n not in allow] - stale = [n for n in allow if n not in need] + gone = [n for n in allow if n not in need] + idle = [n for n in need if n in allow and found[n]] for name in drift: print("tests_referenced: %s is referenced by nothing that can run it -- register it (add_test in " "CMakeLists.txt or cmake/*.cmake), or list it in %s with the reason it must not be a row" % (name, ALLOW)) - for name in stale: + for name in gone: print("tests_referenced: %s is in %s but no such test exists any more -- drop the line" % (name, ALLOW)) - if drift: + for name in idle: + print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing" + % (name, ALLOW)) + if drift or gone or idle: return 1 print("tests_referenced: all %d test script(s) are referenced%s" % ( len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))))) @@ -159,10 +166,20 @@ def root_with(files): str(ALLOW): "keeper.sh # an operator measurement: always exits 0, so a row would be green noise\n"}) cases.append(("an allow-listed test is not drift", check(r), 0, None)) - # 7. a stale allow entry -> reported, but not a failure + # 7. an allow entry for a test that is gone -> a failure, because a list that only gets WARNED about grows r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "tests/one.sh\n", str(ALLOW): "gone.sh # removed long ago\n"}) - cases.append(("a stale allow entry is progress, not drift", check(r), 0, "gone.sh")); cases_root["a stale allow entry is progress, not drift"] = r + cases.append(("an allow entry for a deleted test fails", check(r), 1, "gone.sh")); cases_root["an allow entry for a deleted test fails"] = r + + # 8. an allow entry for a test that IS referenced -> a failure: it suppresses nothing, which is how the list rots + r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n", + str(ALLOW): "one.sh # but it is registered\n"}) + cases.append(("an allow entry that suppresses nothing fails", check(r), 1, "one.sh")); cases_root["an allow entry that suppresses nothing fails"] = r + + # 9. the entry may be written either way: path form and basename mean the same line + r = root_with({"tests/keeper2.sh": "#!/bin/sh\n", "CMakeLists.txt": "", + str(ALLOW): "tests/keeper2.sh # path form is accepted\n"}) + cases.append(("a path-form allow entry is accepted", check(r), 0, None)) # 8. no tests/ at all -> cannot tell, not "clean" r = root_with({"CMakeLists.txt": ""})