From dff38fb0787e4270fbfd72a81fa8d901ff221c06 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 01:51:40 -0400 Subject: [PATCH 1/3] pr_review_queue: read the sha-first verdict shape, which four verdicts on one night were invisible in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The queue only recognised a line that STARTS with the word VERDICT (`VERDICT: READY at `abc1234``). The shape lanes actually write just as often puts the head first: 79e7b2e READY — the arm that was missing is there c0cefd5 CHANGES — one claim in the description has no arm `verdict_kind` answered None for those, so `newest_verdict` fell back to the newest line it COULD read and reported `UNATTRIBUTED`, and the PR stayed on the queue. Measured on four verdicts posted tonight: #2492 showed `newest=UNATTRIBUTED 05:27Z` while a READY sat at its head from 05:48Z, so the queue asked for a review that already existed - the exact duplicate-verdict waste the cap of two per head exists to prevent. Two changes: * VERDICT_LINE / VERDICT_KIND accept `^<7-40 hex>[ \t]+(VERDICT[ \t]+)?`. The head must be a real abbreviated sha and the kind must be one of the known words, so `3 arms, 0 failed` and `abc1234 needs-review` are still not verdicts (both pinned). * `verdict_kind` drops the marker word VERDICT instead of skipping the FIRST token. In the canonical shape the first token is the marker; in the sha-first shape it is the kind, so skipping it turned every sha-first verdict into OTHER. Before/after on the live queue, same minute: before: #2492 head=79e7b2e newest=UNATTRIBUTED 05:27:11Z -> queue: 1 open PR(s) with no verdict after: #2492 head=79e7b2e newest=READY 05:48:17Z -> queue: every open PR has a verdict at its head Arms added to the --state suite (each run): a sha-first READY counts and --state reports it as READY, not UNATTRIBUTED; sha-first CHANGES; the ASCII-hyphen form; a sha-first verdict at a DIFFERENT head still does not count; a number-led line and a sha followed by a non-verdict word are not verdicts. Mutations, each caught (and the first attempt at one was a SyntaxError, i.e. a mutation that tested nothing - redone as valid code): sha-first alternative removed from VERDICT_LINE -> "FAIL a sha-first READY naming this head counts: want True, got False" verdict_kind back to words[1:] -> 4 arms fail (OTHER where READY/CHANGES was wanted) --- tools/ci/pr_review_queue.py | 42 +++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tools/ci/pr_review_queue.py b/tools/ci/pr_review_queue.py index 554ac132f..d764c46c0 100755 --- a/tools/ci/pr_review_queue.py +++ b/tools/ci/pr_review_queue.py @@ -79,10 +79,23 @@ def gh(path, paginate=False): # `abc1234` ``) and quote **other people's** verdicts when reconciling them, and both were being read as verdicts by the # author. Measured 2026-11-14: this lane's own reconciliation comment on npu-kernels#70 quoted `VERDICT: NEEDS-CHANGE` at # the head, so `--state` reported a READY PR as held by a CHANGES that no reviewer had written. -VERDICT_LINE = re.compile(r'^(?![^A-Za-z0-9]*`[^`]*VERDICT)[^A-Za-z0-9]*VERDICT\b', re.MULTILINE) - - -VERDICT_KIND = re.compile(r'^(?![^A-Za-z0-9]*`[^`]*VERDICT)[^A-Za-z0-9]*VERDICT\b[^\n]*', re.MULTILINE) +# Two line shapes, because both are written and only one was read (measured 2026-11-15): the canonical +# VERDICT: READY at `abc1234` +# and the sha-first +# 79e7b2e READY — the arm that was missing is there +# where the head comes first and the verdict word right after it, with no `VERDICT` word anywhere. Four verdicts of +# the second shape were posted on one night's queue and every one of them was invisible: `verdict_kind` answered None, +# so the PR stayed on the queue and the review was repeated by the next lane. The queue's whole claim is that it knows +# what has been reviewed; a convention it cannot read is not a convention. +# +# The sha-first shape requires the head to be a real abbreviated sha (7-40 hex) followed by a known verdict word, so a +# prose line that starts with a number ("3 arms, 0 failed") or a commit subject cannot be mistaken for a verdict. +_SHA_FIRST = r'^(?=([0-9a-f]{7,40}[ \t]+(?:VERDICT[ \t]+)?(?:READY|APPROVE|APPROVED|CHANGES|CHANGES_REQUESTED|CHANGES-REQUESTED|CHANGE|NEEDS-CHANGE|NEEDS-CHANGES|REQUEST-CHANGES|NOT-READY)\b))' +VERDICT_LINE = re.compile( + r'^(?![^A-Za-z0-9]*`[^`]*VERDICT)[^A-Za-z0-9]*VERDICT\b' + '|' + _SHA_FIRST, re.MULTILINE) +VERDICT_KIND = re.compile( + r'^(?![^A-Za-z0-9]*`[^`]*VERDICT)[^A-Za-z0-9]*VERDICT\b[^\n]*' + + '|' + r'^[0-9a-f]{7,40}[ \t]+(?:VERDICT[ \t]+)?[A-Z][A-Z_-]+[^\n]*', re.MULTILINE) def verdict_kind(body): @@ -95,8 +108,10 @@ def verdict_kind(body): found = VERDICT_KIND.search(body or '') if not found: return None - words = re.findall(r'[A-Z][A-Z_-]+', found.group(0)) - for word in words[1:]: + # `VERDICT` is the marker word, not a kind: drop it rather than skipping the first token, because in the sha-first + # shape the first token IS the kind (`79e7b2e READY`). + words = [w for w in re.findall(r'[A-Z][A-Z_-]+', found.group(0)) if w != 'VERDICT'] + for word in words: if word in ('READY', 'APPROVE', 'APPROVED'): return 'READY' # `NEEDS-CHANGE`, `CHANGES-REQUESTED`, `NOT-READY`: a lane writes the verdict in prose as often as in the @@ -284,6 +299,21 @@ def chk(want, got, what): chk('OTHER', verdict_kind('VERDICT: something nobody has seen before'), 'an unknown kind is OTHER, never a silent none') + # The sha-first shape, measured on the live queue 2026-11-15: four verdicts written `79e7b2e READY — ` were + # invisible, so the PR stayed on the queue and the next lane reviewed it again. Every arm below is one of the four + # that were posted that night, plus the lines that must NOT be read as a verdict. + sha_first = [{'body': '79e7b2e READY — the arm that was missing is there', 'submitted_at': '2026-11-15T05:48:17Z'}] + chk(True, reviewed_at(sha_first, '79e7b2e'), 'a sha-first READY naming this head counts') + chk(('READY', '2026-11-15T05:48:17Z'), newest_verdict(sha_first, [], '79e7b2e'), + 'and --state reports it as READY, not UNATTRIBUTED') + chk('READY', verdict_kind('79e7b2e READY — the arm is there'), 'the sha-first kind is read, not skipped') + chk('CHANGES', verdict_kind('c0cefd5 CHANGES — one claim has no arm'), 'sha-first CHANGES too') + chk('CHANGES', verdict_kind('7f7cfff CHANGES - a hyphen, not an em dash'), 'the ASCII-hyphen form too') + chk(False, reviewed_at([{'body': '79e7b2e READY — x'}], '9999999'), + 'a sha-first verdict at a DIFFERENT head still does not count') + chk(None, verdict_kind('3 arms, 0 failed'), 'a line that starts with a number is not a verdict') + chk(None, verdict_kind('abc1234 needs-review'), 'a known sha followed by a non-verdict word is not a verdict') + # the attributed case must still prefer the newest NAMED verdict revs2 = [{'body': 'VERDICT: CHANGES at `abc1234`', 'submitted_at': '2026-11-14T03:00:00Z'}, {'body': 'VERDICT: READY at `abc1234`', 'submitted_at': '2026-11-14T03:10:00Z'}] From 3539394f23ec22eecec66447e1c8c6e437088be0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 02:45:56 -0400 Subject: [PATCH 2/3] pr_review_queue: one verdict vocabulary, so the tool stops disagreeing with itself (review note on #2495) The sha-first alternative was written twice: VERDICT_LINE restricted the word after the sha to the verdict vocabulary, VERDICT_KIND accepted ANY capitalised word. So a comment that merely QUOTED a failing build here is what the log said: 79e7b2e FAILED - the build broke was not a VERDICT_LINE but WAS a verdict of kind OTHER - and `newest_verdict` gates on verdict_kind alone, so `--state` would report that PR as reviewed while `reviewed_at` reported it as not reviewed. Disagreeing with yourself is worse than either answer: one of the two would be used to decide whether to review again. Both regexes are now built from one `_KINDS` alternation. Consequences, each pinned by an arm: 79e7b2e READY - x line yes kind READY (unchanged) abc1234 FAILED - the build broke line no kind None (was OTHER) dff38fb0 pr_review_queue: read the sha-first line no kind None VERDICT: something nobody has seen before line yes kind OTHER (unchanged, deliberately: an unknown kind after the VERDICT word must not fall through to an older READY) Mutation: put the broad `[A-Z][A-Z_-]+` back in VERDICT_KIND's sha-first half (valid code, verified importing) -> FAIL a quoted FAILED line is not a verdict kind: want None, got 'OTHER' FAIL and --state does not count it as a review at all: want None, got ('OTHER', '2026-11-15T05:00:00Z') Unmutated: --state --self-test ok; --self-test ok; the live six-repo queue is unchanged. --- tools/ci/pr_review_queue.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/ci/pr_review_queue.py b/tools/ci/pr_review_queue.py index d764c46c0..115588bde 100755 --- a/tools/ci/pr_review_queue.py +++ b/tools/ci/pr_review_queue.py @@ -90,12 +90,19 @@ def gh(path, paginate=False): # # The sha-first shape requires the head to be a real abbreviated sha (7-40 hex) followed by a known verdict word, so a # prose line that starts with a number ("3 arms, 0 failed") or a commit subject cannot be mistaken for a verdict. -_SHA_FIRST = r'^(?=([0-9a-f]{7,40}[ \t]+(?:VERDICT[ \t]+)?(?:READY|APPROVE|APPROVED|CHANGES|CHANGES_REQUESTED|CHANGES-REQUESTED|CHANGE|NEEDS-CHANGE|NEEDS-CHANGES|REQUEST-CHANGES|NOT-READY)\b))' +# ONE vocabulary, used by both regexes. The first version of this fix had the sha-first alternative written twice: +# VERDICT_LINE restricted the word after the sha to the verdict vocabulary while VERDICT_KIND accepted ANY capitalised +# word. So `abc1234 FAILED - the build broke` (a commit subject, or a quoted log line) was not a VERDICT_LINE but WAS a +# verdict of kind OTHER, and `newest_verdict` - which gates on verdict_kind only - would call a PR reviewed on a comment +# that merely quoted a failing build. The tool disagreed with itself, which is worse than either answer alone. +_KINDS = (r'READY|APPROVE|APPROVED|CHANGES|CHANGES_REQUESTED|CHANGES-REQUESTED|CHANGE|NEEDS-CHANGE|NEEDS-CHANGES|' + r'REQUEST-CHANGES|NOT-READY') +_SHA_FIRST = r'^(?=([0-9a-f]{7,40}[ \t]+(?:VERDICT[ \t]+)?(?:' + _KINDS + r')\b))' VERDICT_LINE = re.compile( r'^(?![^A-Za-z0-9]*`[^`]*VERDICT)[^A-Za-z0-9]*VERDICT\b' + '|' + _SHA_FIRST, re.MULTILINE) VERDICT_KIND = re.compile( r'^(?![^A-Za-z0-9]*`[^`]*VERDICT)[^A-Za-z0-9]*VERDICT\b[^\n]*' + - '|' + r'^[0-9a-f]{7,40}[ \t]+(?:VERDICT[ \t]+)?[A-Z][A-Z_-]+[^\n]*', re.MULTILINE) + '|' + r'^[0-9a-f]{7,40}[ \t]+(?:VERDICT[ \t]+)?(?:' + _KINDS + r')\b[^\n]*', re.MULTILINE) def verdict_kind(body): @@ -314,6 +321,16 @@ def chk(want, got, what): chk(None, verdict_kind('3 arms, 0 failed'), 'a line that starts with a number is not a verdict') chk(None, verdict_kind('abc1234 needs-review'), 'a known sha followed by a non-verdict word is not a verdict') + # The tool must not disagree with itself. A commit subject or a QUOTED log line that starts with a sha is not a + # verdict, and `newest_verdict` gates on verdict_kind alone -- so a broad verdict_kind would let a quoted `FAILED` + # mark a PR reviewed while `reviewed_at` says it is not. + quoted = [{'body': 'here is what the log said:\n\n79e7b2e FAILED - the build broke\n', 'created_at': '2026-11-15T05:00:00Z'}] + chk(None, verdict_kind('79e7b2e FAILED - the build broke'), 'a quoted FAILED line is not a verdict kind') + chk(None, newest_verdict([], quoted, '79e7b2e'), 'and --state does not count it as a review at all') + chk(False, reviewed_at(quoted, '79e7b2e'), 'which is what reviewed_at already said - the two now agree') + chk(None, verdict_kind('dff38fb0 pr_review_queue: read the sha-first verdict shape'), + 'a commit subject is not a verdict either') + # the attributed case must still prefer the newest NAMED verdict revs2 = [{'body': 'VERDICT: CHANGES at `abc1234`', 'submitted_at': '2026-11-14T03:00:00Z'}, {'body': 'VERDICT: READY at `abc1234`', 'submitted_at': '2026-11-14T03:10:00Z'}] From 0a11b721bf193f928328446251c50e05aef508bf Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 03:06:08 -0400 Subject: [PATCH 3/3] pr_review_queue: one --self-test flag runs both suites, because CI runs exactly one flag (review note on #2495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool had two self-tests: `self_test()` (12 arms) and `self_test_state()` (16 arms), and `--state` SELECTED between them instead of adding to them. The ctest row is add_test(NAME pr_review_queue_selftest COMMAND python3 … --self-test) with no `--state` anywhere in CMakeLists.txt, cmake/ or .github/workflows/, so **the 16 arms about the shapes that made `--state` lie were never run in CI** -- including the ones from tonight's two fixes. A check the row does not run is not a check, and this is my own tool: the review that caught it did so by trying to verify my arms with the wrong flag and getting a green "ok" twice. Now `--self-test` runs both, `--state --self-test` still works, and each suite prints its arm count (`12 arms`, `16 arms`) with one closing verdict line for the pair, because the per-suite lines print in order and a reader who sees "ok (12 arms)" followed by a FAIL from the second suite needs the last line to be the answer: pr_review_queue --self-test: ok (12 arms) pr_review_queue --state self-test: ok (16 arms) pr_review_queue --self-test: ok (both suites) Measured, the mutation that used to pass the CI row (broad `[A-Z][A-Z_-]+` restored in VERDICT_KIND's sha-first half, verified importing before judging it): before this commit: --self-test -> "ok (12 arms)" the state arms never ran after: --self-test -> FAIL a quoted FAILED line is not a verdict kind: want None, got 'OTHER' FAIL and --state does not count it as a review at all: want None, got ('OTHER', …) pr_review_queue --self-test: FAIL (both suites) rc 1 Clean tree: rc 0 from the exact command the ctest row runs. Live six-repo queue unchanged. --- tools/ci/pr_review_queue.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/ci/pr_review_queue.py b/tools/ci/pr_review_queue.py index 115588bde..37704d1c1 100755 --- a/tools/ci/pr_review_queue.py +++ b/tools/ci/pr_review_queue.py @@ -209,9 +209,11 @@ def queue(repos, min_age_hours=0.0, now_epoch=None): def self_test(): rc = 0 + arms = [0] def chk(want, got, what): nonlocal rc + arms[0] += 1 if want == got: print(f"ok {what}") else: @@ -263,7 +265,7 @@ def chk(want, got, what): 'a verdict with markdown decoration before it still counts') chk(True, reviewed_at([{'body': 'VERDICT: CHANGES at `abc1234`'}], 'abc1234'), 'CHANGES is a verdict: the PR was reviewed, the answer was no') - print(f"pr_review_queue --self-test: {'ok' if rc == 0 else 'FAIL'}") + print(f"pr_review_queue --self-test: {'ok' if rc == 0 else 'FAIL'} ({arms[0]} arms)") return rc @@ -293,8 +295,10 @@ def state(repos): def self_test_state(): """The two shapes that made `--state` lie until they were run against the live queue.""" problems = [] + arms = [0] def chk(want, got, what): + arms[0] += 1 if want != got: problems.append(f"{what}: want {want!r}, got {got!r}") @@ -341,7 +345,7 @@ def chk(want, got, what): for problem in problems: print('FAIL ' + problem) return 1 - print('pr_review_queue --state self-test: ok') + print(f'pr_review_queue --state self-test: ok ({arms[0]} arms)') return 0 @@ -442,7 +446,16 @@ def main(argv): if a.self_test: if a.duplicates: return self_test_duplicates() - return self_test() if not a.state else self_test_state() + # Both suites, always. `--state` used to SELECT one of them, so the row CI runs (`--self-test`, no flag) never + # executed the 16 assertions about the shapes that made `--state` lie -- a check the row does not run is not a + # check. `--state --self-test` still works and runs the same two; the flag now only says which output the caller + # expected, not which half to skip. + rc_main = self_test() + rc_state = self_test_state() + # One verdict for both halves: the per-suite lines print in order, so a reader who sees "ok (12 arms)" and then a + # FAIL from the second suite needs the last line to say the run failed. + print(f"pr_review_queue --self-test: {'ok' if not (rc_main or rc_state) else 'FAIL'} (both suites)") + return rc_main or rc_state if a.duplicates: rows = duplicates(a.repos) if a.json: