Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,11 @@ add_test(NAME doc_section_loss_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tool
# the class in CI. Files whose `#if` conditions cannot be resolved are named as not checked, never guessed at.
add_test(NAME lane_syntax_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/lane_syntax_check.py --self-test
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
# The review queue every reviewer hand-rolls with `gh api`, which defaults to per_page=30 -- a PR with 30+ reviews
# loses its newest ones, so a verdict that exists reads as missing and a reviewer repeats work. Only the SELF-TEST is a
# row: the live mode needs `gh` and the network, which a ctest row must not.
add_test(NAME pr_review_queue_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/pr_review_queue.py --self-test
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
add_dependencies(onebit npu_decode_note_test)
# engine#2106: a shipped kernel set's own consistency -- manifest vs bytes, one form or none, name vs directory. The
# refusal this pre-empts ("differ in size") needed private-repo archaeology to diagnose; the symptom is checkable here.
Expand Down
180 changes: 180 additions & 0 deletions tools/ci/pr_review_queue.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""pr_review_queue.py -- which open PRs still need a verdict at their CURRENT head.

A review is a claim about a head. So the question is not "has this PR been reviewed" but "does any review name the head
this PR has now" -- and a PR that has moved since its verdict is back in the queue, which is the whole point of pinning
the 7-hex head in the verdict line.

Two things this gets right that a hand-rolled `gh api` loop gets wrong, both of them measured tonight:

* **`gh api` defaults to `per_page=30`.** A PR with 30 reviews silently loses the newest ones, so a verdict that
exists reads as missing -- and a reviewer repeats work, or asserts on the record that a verdict was never given.
Both list endpoints here page until exhausted.
* **Merged and draft PRs are not in the queue.** A draft is not asking for a verdict, and a merged PR cannot take one.

Usage:
pr_review_queue.py # the fleet's six repos
pr_review_queue.py --repos engine site # a subset
pr_review_queue.py --min-age-hours 1 # only those past the 1 h gate
pr_review_queue.py --json
pr_review_queue.py --self-test
"""

import argparse
import json
import re
import subprocess
import sys

DEFAULT_REPOS = ('engine', 'llama.cpp', 'gpu-kernels', 'npu-kernels', 'lemonade', 'site')
OWNER = '1bit-MONSTER'
PAGE = 100


def gh(path, paginate=False):
"""One page, or every page, of a `gh api` list endpoint."""
if not paginate:
return subprocess.run(['gh', 'api', path], capture_output=True, text=True).stdout
out, page = [], 1
while True:
sep = '&' if '?' in path else '?'
chunk = subprocess.run(['gh', 'api', f"{path}{sep}per_page={PAGE}&page={page}"],
capture_output=True, text=True).stdout
try:
got = json.loads(chunk or '[]')
except json.JSONDecodeError:
break
if not got:
break
out.extend(got)
if len(got) < PAGE:
break
page += 1
return json.dumps(out)


VERDICT_LINE = re.compile(r'^[^A-Za-z0-9]*VERDICT\b', re.MULTILINE)


def reviewed_at(reviews, head):
"""True if any review OR COMMENT body carries a VERDICT line naming this head prefix.

Two things this has to get right, both learned from a review of this tool:

* a verdict posted as an **issue comment** counts. Most lanes put `VERDICT: ...` in a comment, not in a review
object -- measured on the open queue, four PRs (llama.cpp#280, site#17/#16/#15) had a comment verdict and no
review at all, so a reviews-only queue asked for reviews that already existed;
* the token alone is not a verdict: `no VERDICT at abc1234 yet` contains both the word and the sha and means the
opposite. The verdict has to START a line (markdown decoration before it is fine), which is the convention the
lanes actually write.

A verdict line carries the 7-hex head (`VERDICT: READY at `abc1234``); a body that names a longer sha covers the
shorter prefix too, which is why this is a substring test on the head rather than an equality on the whole sha.
"""
for r in reviews:
body = r.get('body') or ''
if head in body and VERDICT_LINE.search(body):
return True
return False


def queue(repos, min_age_hours=0.0, now_epoch=None):
"""[{repo, number, head, age_hours, title}] for open, non-draft PRs with no verdict at their head."""
import datetime
now_epoch = now_epoch if now_epoch is not None else datetime.datetime.now(datetime.timezone.utc).timestamp()
out = []
for repo in repos:
for pr in json.loads(gh(f'repos/{OWNER}/{repo}/pulls?state=open', paginate=True) or '[]'):
if pr.get('draft'):
continue
head = (pr.get('head') or {}).get('sha', '')
if not head:
continue
# Both channels: a review body and an issue comment. Same list, because `reviewed_at` asks one question.
revs = json.loads(gh(f"repos/{OWNER}/{repo}/pulls/{pr['number']}/reviews", paginate=True) or '[]')
revs += json.loads(gh(f"repos/{OWNER}/{repo}/issues/{pr['number']}/comments", paginate=True) or '[]')
if reviewed_at(revs, head[:7]):
continue
created = datetime.datetime.fromisoformat(pr['created_at'].replace('Z', '+00:00')).timestamp()
age = (now_epoch - created) / 3600.0
if age < min_age_hours:
continue
out.append({'repo': repo, 'number': pr['number'], 'head': head[:7],
'age_hours': round(age, 2), 'title': (pr.get('title') or '')[:48]})
return sorted(out, key=lambda r: -r['age_hours'])


def self_test():
rc = 0

def chk(want, got, what):
nonlocal rc
if want == got:
print(f"ok {what}")
else:
print(f"FAIL {what}: want {want}, got {got}")
rc = 1

chk(True, reviewed_at([{'body': 'VERDICT: READY at `abc1234`'}], 'abc1234'), 'a verdict naming the head counts')
chk(True, reviewed_at([{'body': 'VERDICT: READY at `abc1234def`'}], 'abc1234'),
'a verdict naming a longer sha of the same head counts too')
chk(False, reviewed_at([{'body': 'VERDICT: READY at `9999999`'}], 'abc1234'),
'a verdict at a DIFFERENT head does not count: that is the case the queue exists for')
chk(False, reviewed_at([{'body': 'looks good to me'}], 'abc1234'),
'a review without a verdict line does not count')
# The guard is the VERDICT word, not the head: removing it left --self-test at rc 0, because every arm above
# passed a body that contained BOTH. A comment that quotes the head while reviewing is the shape that separates
# them, and it is the common one -- reviewers quote the sha they looked at.
chk(False, reviewed_at([{'body': 'I read abc1234 carefully and it looks fine to me'}], 'abc1234'),
'a comment that NAMES the head but is not a verdict does not count')
# The review of this tool found the arm above too weak: it carries no VERDICT token at all, so it passes even
# when the matcher is `'VERDICT' in body`. This one has both tokens and still means the opposite.
chk(False, reviewed_at([{'body': 'no VERDICT at abc1234 yet'}], 'abc1234'),
'a sentence that DISCLAIMS a verdict is not a verdict')
chk(False, reviewed_at([{'body': None}], 'abc1234'), 'a null body does not raise')
chk(True, reviewed_at([{'body': '**VERDICT: READY** `abc1234`'}], 'abc1234'),
'a verdict with markdown decoration before it still counts')
chk(True, reviewed_at([{'body': 'VERDICT: CHANGES at `abc1234`'}], 'abc1234'),
'CHANGES is a verdict: the PR was reviewed, the answer was no')
print(f"pr_review_queue --self-test: {'ok' if rc == 0 else 'FAIL'}")
return rc


def main(argv):
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument('--repos', nargs='+', default=list(DEFAULT_REPOS))
ap.add_argument('--min-age-hours', type=float, default=0.0)
ap.add_argument('--json', action='store_true')
ap.add_argument('--self-test', action='store_true')
a = ap.parse_args(argv)
if a.self_test:
return self_test()
q = queue(a.repos, a.min_age_hours)
if a.json:
print(json.dumps(q, indent=1))
return 0
if not q:
print('pr_review_queue: every open PR has a verdict at its current head')
return 0
for r in q:
print(f" {r['repo']:12} #{r['number']:<5} {r['age_hours']:6.2f}h head={r['head']} {r['title']}")
print(f"pr_review_queue: {len(q)} open PR(s) with no verdict at their current head")
return 0


if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
Loading