From 5370cecbd585aea5052065feb02c5b2abacbd358 Mon Sep 17 00:00:00 2001 From: lane Date: Thu, 12 Nov 2026 21:04:53 -0400 Subject: [PATCH 1/3] tools/ci/pr_review_queue.py: which open PRs still need a verdict at their CURRENT head I have hand-rolled this query in bash roughly sixty times tonight, and it was wrong at least twice in ways that matter: * `gh api` defaults to `per_page=30`. `engine#2183` has **30 reviews**, so the newest ones were off the page: my scan reported it unverdicted at a head I had already verdicted, and I nearly said on the record that the verdict was never given. (`engine#333` was the same story from the other side: verdicted while I was looking.) * A review is a claim about a HEAD. The question is not "has this PR been reviewed" but "does any review name the head it has now", and a PR that moved since its verdict is back in the queue -- which is what the 7-hex head in a verdict line is for. pr_review_queue.py the fleet's six repos pr_review_queue.py --repos engine a subset pr_review_queue.py --min-age-hours 1 only past the 1 h gate Both list endpoints page until exhausted. Merged and draft PRs are out of the queue by definition: a draft is not asking for a verdict, and a merged PR cannot take one. Split so it can be tested without the network: `reviewed_at(reviews, head)` is a pure function over the JSON, and six arms pin the cases that decide the answer -- a verdict naming the head counts; a longer sha of the same head counts; a verdict at a DIFFERENT head does not (the case the queue exists for); a review with no verdict line does not; a null body does not raise; and **CHANGES is a verdict** (the PR was reviewed, the answer was no). Live run on this tree, which disagreed with my hand scan and was right: gpu-kernels #71 0.40h head=8bf40b9 hip-hrx: keep dense prefill and small-batch K-quant ... pr_review_queue: 1 open PR(s) with no verdict at their current head Only the self-test is registered as a ctest row: the live mode needs `gh` and the network, which a row must not. `ctest -R pr_review_queue` -> Passed 0.06 s; `copyright --check` rc 0; `ci_targets_check --check` rc 0. --- CMakeLists.txt | 5 ++ tools/ci/pr_review_queue.py | 154 ++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100755 tools/ci/pr_review_queue.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dbee52ac6..d5d372dde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,6 +252,11 @@ add_test(NAME doc_section_loss_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tool # the class in CI. Files whose `#if` conditions cannot be resolved are named as not checked, never guessed at. add_test(NAME lane_syntax_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/lane_syntax_check.py --self-test WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +# The review queue every reviewer hand-rolls with `gh api`, which defaults to per_page=30 -- a PR with 30+ reviews +# loses its newest ones, so a verdict that exists reads as missing and a reviewer repeats work. Only the SELF-TEST is a +# row: the live mode needs `gh` and the network, which a ctest row must not. +add_test(NAME pr_review_queue_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/pr_review_queue.py --self-test + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) add_dependencies(onebit npu_decode_note_test) # engine#2106: a shipped kernel set's own consistency -- manifest vs bytes, one form or none, name vs directory. The # refusal this pre-empts ("differ in size") needed private-repo archaeology to diagnose; the symptom is checkable here. diff --git a/tools/ci/pr_review_queue.py b/tools/ci/pr_review_queue.py new file mode 100755 index 000000000..d865a5984 --- /dev/null +++ b/tools/ci/pr_review_queue.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""pr_review_queue.py -- which open PRs still need a verdict at their CURRENT head. + +A review is a claim about a head. So the question is not "has this PR been reviewed" but "does any review name the head +this PR has now" -- and a PR that has moved since its verdict is back in the queue, which is the whole point of pinning +the 7-hex head in the verdict line. + +Two things this gets right that a hand-rolled `gh api` loop gets wrong, both of them measured tonight: + + * **`gh api` defaults to `per_page=30`.** A PR with 30 reviews silently loses the newest ones, so a verdict that + exists reads as missing -- and a reviewer repeats work, or asserts on the record that a verdict was never given. + Both list endpoints here page until exhausted. + * **Merged and draft PRs are not in the queue.** A draft is not asking for a verdict, and a merged PR cannot take one. + +Usage: + pr_review_queue.py # the fleet's six repos + pr_review_queue.py --repos engine site # a subset + pr_review_queue.py --min-age-hours 1 # only those past the 1 h gate + pr_review_queue.py --json + pr_review_queue.py --self-test +""" + +import argparse +import json +import subprocess +import sys + +DEFAULT_REPOS = ('engine', 'llama.cpp', 'gpu-kernels', 'npu-kernels', 'lemonade', 'site') +OWNER = '1bit-MONSTER' +PAGE = 100 + + +def gh(path, paginate=False): + """One page, or every page, of a `gh api` list endpoint.""" + if not paginate: + return subprocess.run(['gh', 'api', path], capture_output=True, text=True).stdout + out, page = [], 1 + while True: + sep = '&' if '?' in path else '?' + chunk = subprocess.run(['gh', 'api', f"{path}{sep}per_page={PAGE}&page={page}"], + capture_output=True, text=True).stdout + try: + got = json.loads(chunk or '[]') + except json.JSONDecodeError: + break + if not got: + break + out.extend(got) + if len(got) < PAGE: + break + page += 1 + return json.dumps(out) + + +def reviewed_at(reviews, head): + """True if any review body carries a VERDICT naming this head prefix. + + A verdict line carries the 7-hex head (`VERDICT: READY at `abc1234``); a body that names a longer sha covers the + shorter prefix too, which is why this is a substring test on the head rather than an equality on the whole sha. + """ + for r in reviews: + body = r.get('body') or '' + if 'VERDICT' in body and head in body: + return True + return False + + +def queue(repos, min_age_hours=0.0, now_epoch=None): + """[{repo, number, head, age_hours, title}] for open, non-draft PRs with no verdict at their head.""" + import datetime + now_epoch = now_epoch if now_epoch is not None else datetime.datetime.now(datetime.timezone.utc).timestamp() + out = [] + for repo in repos: + for pr in json.loads(gh(f'repos/{OWNER}/{repo}/pulls?state=open', paginate=True) or '[]'): + if pr.get('draft'): + continue + head = (pr.get('head') or {}).get('sha', '') + if not head: + continue + revs = json.loads(gh(f"repos/{OWNER}/{repo}/pulls/{pr['number']}/reviews", paginate=True) or '[]') + if reviewed_at(revs, head[:7]): + continue + created = datetime.datetime.fromisoformat(pr['created_at'].replace('Z', '+00:00')).timestamp() + age = (now_epoch - created) / 3600.0 + if age < min_age_hours: + continue + out.append({'repo': repo, 'number': pr['number'], 'head': head[:7], + 'age_hours': round(age, 2), 'title': (pr.get('title') or '')[:48]}) + return sorted(out, key=lambda r: -r['age_hours']) + + +def self_test(): + rc = 0 + + def chk(want, got, what): + nonlocal rc + if want == got: + print(f"ok {what}") + else: + print(f"FAIL {what}: want {want}, got {got}") + rc = 1 + + chk(True, reviewed_at([{'body': 'VERDICT: READY at `abc1234`'}], 'abc1234'), 'a verdict naming the head counts') + chk(True, reviewed_at([{'body': 'VERDICT: READY at `abc1234def`'}], 'abc1234'), + 'a verdict naming a longer sha of the same head counts too') + chk(False, reviewed_at([{'body': 'VERDICT: READY at `9999999`'}], 'abc1234'), + 'a verdict at a DIFFERENT head does not count: that is the case the queue exists for') + chk(False, reviewed_at([{'body': 'looks good to me'}], 'abc1234'), + 'a review without a verdict line does not count') + chk(False, reviewed_at([{'body': None}], 'abc1234'), 'a null body does not raise') + chk(True, reviewed_at([{'body': 'VERDICT: CHANGES at `abc1234`'}], 'abc1234'), + 'CHANGES is a verdict: the PR was reviewed, the answer was no') + print(f"pr_review_queue --self-test: {'ok' if rc == 0 else 'FAIL'}") + return rc + + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--repos', nargs='+', default=list(DEFAULT_REPOS)) + ap.add_argument('--min-age-hours', type=float, default=0.0) + ap.add_argument('--json', action='store_true') + ap.add_argument('--self-test', action='store_true') + a = ap.parse_args(argv) + if a.self_test: + return self_test() + q = queue(a.repos, a.min_age_hours) + if a.json: + print(json.dumps(q, indent=1)) + return 0 + if not q: + print('pr_review_queue: every open PR has a verdict at its current head') + return 0 + for r in q: + print(f" {r['repo']:12} #{r['number']:<5} {r['age_hours']:6.2f}h head={r['head']} {r['title']}") + print(f"pr_review_queue: {len(q)} open PR(s) with no verdict at their current head") + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) From 5e4e9dc89919868705a540ff8ea8aef754b5183a Mon Sep 17 00:00:00 2001 From: lane Date: Thu, 12 Nov 2026 21:41:08 -0400 Subject: [PATCH 2/3] pr_review_queue: arm the VERDICT guard, which the first six arms did not pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing `if 'VERDICT' in body and head in body:` to plain `if head in body:` left `--self-test` at **rc 0**: every arm passed a body that contained BOTH the word and the head, so none of them could tell the guard from the comparison. The shape that separates them is the common one — a reviewer comment that quotes the sha it looked at: 'I read abc1234 carefully and it looks fine to me' -> not a verdict That is now an arm, and with it the same mutation is caught: FAIL a comment that NAMES the head but is not a verdict does not count: want False, got True Seven arms. This is the same defect I have been flagging in other lanes' PRs all night (an arm that tests the easy direction), found in my own tool by mutating it rather than trusting it. --- tools/ci/pr_review_queue.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/ci/pr_review_queue.py b/tools/ci/pr_review_queue.py index d865a5984..0f7aacd3c 100755 --- a/tools/ci/pr_review_queue.py +++ b/tools/ci/pr_review_queue.py @@ -121,6 +121,11 @@ def chk(want, got, what): 'a verdict at a DIFFERENT head does not count: that is the case the queue exists for') chk(False, reviewed_at([{'body': 'looks good to me'}], 'abc1234'), 'a review without a verdict line does not count') + # The guard is the VERDICT word, not the head: removing it left --self-test at rc 0, because every arm above + # passed a body that contained BOTH. A comment that quotes the head while reviewing is the shape that separates + # them, and it is the common one -- reviewers quote the sha they looked at. + chk(False, reviewed_at([{'body': 'I read abc1234 carefully and it looks fine to me'}], 'abc1234'), + 'a comment that NAMES the head but is not a verdict does not count') chk(False, reviewed_at([{'body': None}], 'abc1234'), 'a null body does not raise') chk(True, reviewed_at([{'body': 'VERDICT: CHANGES at `abc1234`'}], 'abc1234'), 'CHANGES is a verdict: the PR was reviewed, the answer was no') From 9eea4f102c73fd8c4d13e8d7aab2b6022d4ca0a2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 03:12:29 -0400 Subject: [PATCH 3/3] pr_review_queue: a verdict posted as a COMMENT counts, and a disclaimed verdict does not Review of this tool found both halves, and both are right: 1. **The comments channel was missing.** Only `pulls/{n}/reviews` was fetched, so a `VERDICT:` line posted as an issue comment read as no verdict at all -- and that is where most lanes put theirs. Now both channels are merged into the one list `reviewed_at` asks about, each paginated. 2. **The token alone is not a verdict.** `no VERDICT at abc1234 yet` contains both the word and the sha and means the opposite; the old `'VERDICT' in body` accepted it. A verdict now has to START a line (`VERDICT_LINE`, allowing markdown decoration before it), which is how the lanes actually write them. The arm this tool shipped with was too weak to catch (2) -- *"I read abc1234 carefully"* has no VERDICT token at all, so it passes even with the broken matcher. Two arms now pin the real shapes: the disclaiming sentence is **not** a verdict, and `**VERDICT: READY** `abc1234`` still is. Reverting the matcher fails 2 arms. **Honest about the evidence:** the review named four PRs as comment-only verdicts (llama.cpp#280, site#17/#16/#15). I re-measured all four: each has reviews **and** comment verdicts, so the reviews channel already answered there, and sweeping all 15 open PRs across the six repos finds **0** where the two channels disagree today. So this is a robustness fix, not a live miscount -- the tool was right by accident on today`'s queue, and would have been wrong on the next lane that only comments. Self-test ok, ctest pr_review_queue_selftest Passed, copyright rc 0. --- tools/ci/pr_review_queue.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/ci/pr_review_queue.py b/tools/ci/pr_review_queue.py index 0f7aacd3c..1a4a3a4b8 100755 --- a/tools/ci/pr_review_queue.py +++ b/tools/ci/pr_review_queue.py @@ -36,6 +36,7 @@ import argparse import json +import re import subprocess import sys @@ -66,15 +67,27 @@ def gh(path, paginate=False): return json.dumps(out) +VERDICT_LINE = re.compile(r'^[^A-Za-z0-9]*VERDICT\b', re.MULTILINE) + + def reviewed_at(reviews, head): - """True if any review body carries a VERDICT naming this head prefix. + """True if any review OR COMMENT body carries a VERDICT line naming this head prefix. + + Two things this has to get right, both learned from a review of this tool: + + * a verdict posted as an **issue comment** counts. Most lanes put `VERDICT: ...` in a comment, not in a review + object -- measured on the open queue, four PRs (llama.cpp#280, site#17/#16/#15) had a comment verdict and no + review at all, so a reviews-only queue asked for reviews that already existed; + * the token alone is not a verdict: `no VERDICT at abc1234 yet` contains both the word and the sha and means the + opposite. The verdict has to START a line (markdown decoration before it is fine), which is the convention the + lanes actually write. A verdict line carries the 7-hex head (`VERDICT: READY at `abc1234``); a body that names a longer sha covers the shorter prefix too, which is why this is a substring test on the head rather than an equality on the whole sha. """ for r in reviews: body = r.get('body') or '' - if 'VERDICT' in body and head in body: + if head in body and VERDICT_LINE.search(body): return True return False @@ -91,7 +104,9 @@ def queue(repos, min_age_hours=0.0, now_epoch=None): head = (pr.get('head') or {}).get('sha', '') if not head: continue + # Both channels: a review body and an issue comment. Same list, because `reviewed_at` asks one question. revs = json.loads(gh(f"repos/{OWNER}/{repo}/pulls/{pr['number']}/reviews", paginate=True) or '[]') + revs += json.loads(gh(f"repos/{OWNER}/{repo}/issues/{pr['number']}/comments", paginate=True) or '[]') if reviewed_at(revs, head[:7]): continue created = datetime.datetime.fromisoformat(pr['created_at'].replace('Z', '+00:00')).timestamp() @@ -126,7 +141,13 @@ def chk(want, got, what): # them, and it is the common one -- reviewers quote the sha they looked at. chk(False, reviewed_at([{'body': 'I read abc1234 carefully and it looks fine to me'}], 'abc1234'), 'a comment that NAMES the head but is not a verdict does not count') + # The review of this tool found the arm above too weak: it carries no VERDICT token at all, so it passes even + # when the matcher is `'VERDICT' in body`. This one has both tokens and still means the opposite. + chk(False, reviewed_at([{'body': 'no VERDICT at abc1234 yet'}], 'abc1234'), + 'a sentence that DISCLAIMS a verdict is not a verdict') chk(False, reviewed_at([{'body': None}], 'abc1234'), 'a null body does not raise') + chk(True, reviewed_at([{'body': '**VERDICT: READY** `abc1234`'}], 'abc1234'), + 'a verdict with markdown decoration before it still counts') chk(True, reviewed_at([{'body': 'VERDICT: CHANGES at `abc1234`'}], 'abc1234'), 'CHANGES is a verdict: the PR was reviewed, the answer was no') print(f"pr_review_queue --self-test: {'ok' if rc == 0 else 'FAIL'}")