From 9add17cd20cb5017a12f1b50d41a6664d373e04b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 23:05:10 -0400 Subject: [PATCH 1/2] ci: pr_verdict reads NOT READY, checks the VERDICT argument, and reads back the post by id Three holes reported on #2607 and #2616, each probed against the merged tool: - NOT READY was read as READY: verdict_in_first_line was a substring test over ("READY", "CHANGES"). It is now a whole-word match with NOT READY first, so "READYs" in a note is not a verdict either. - The VERDICT argument was only printed: --post R N body.md READY with a CHANGES body posted the CHANGES and said "posted READY". classify() now refuses the mismatch. - The read-back could not fail on a head that already had a verdict: verdicted_at matched the 7-hex and a verdict word anywhere in any review or comment, so a peer's older verdict, or a "**Not a verdict** (abc1234 has its READYs)" note, proved the post. The post now goes through the REST API, so it returns the new object's id (a review also gets commit_id=head, so it cannot attach to a head that moved). That one object is read back by id, and its first line must classify as the verdict posted; a review must also carry commit_id == head. verdicted_at (used by --check) now counts only a first-line verdict, which is what pr_review_queue reads. The pending-review message no longer offers DELETE as the remedy. The draft is another lane's work; it points at --as-comment instead. Self-test covers each case; three mutants fail it (old regex order, the argument check removed, the commit_id check removed). Live on #2617 (a peer's pending review is open): --check names --as-comment, and --post ... READY --dry-run with a CHANGES body is refused "the first line says CHANGES, the VERDICT argument is READY". Co-Authored-By: Claude Opus 5.5 --- tools/ci/pr_verdict.py | 100 ++++++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/tools/ci/pr_verdict.py b/tools/ci/pr_verdict.py index d1e531335..7e0f87475 100755 --- a/tools/ci/pr_verdict.py +++ b/tools/ci/pr_verdict.py @@ -48,7 +48,8 @@ import sys HEAD_RE = re.compile(r"\b([0-9a-f]{7,40})\b") -VERDICTS = ("READY", "CHANGES") +# NOT READY first: it contains READY, so a substring test in any other order reads it as READY (review of #2607). +VERDICTS = ("NOT READY", "READY", "CHANGES") def first_line_sha(body): @@ -58,16 +59,21 @@ def first_line_sha(body): return m.group(1) if m else None +# whole words, NOT READY first: "READYs" in "(abc1234 has its READYs)" is not a verdict, and NOT READY is not READY +VERDICT_RE = re.compile(r"\b(NOT READY|READY|CHANGES)\b") + + def verdict_in_first_line(body): first = body.splitlines()[0] if body.strip() else "" - for v in VERDICTS: - if v in first: - return v - return None + m = VERDICT_RE.search(first) + return m.group(1) if m else None + +def classify(body, head, verdict=None): + """(ok, reason) for a body against a head -- pure, so the self-test can drive every case. -def classify(body, head): - """(ok, reason) for a body against a head -- pure, so the self-test can drive every case.""" + `verdict`, when given, is the VERDICT argument of --post: a body whose first line says something else is refused, + because that mismatch is exactly the mistake the argument exists to catch (review of #2607).""" if not body.strip(): return False, "empty body" sha = first_line_sha(body) @@ -77,6 +83,9 @@ def classify(body, head): return False, f"the first line names {sha}, the head is {head[:7]}: this would review code that is gone" if verdict_in_first_line(body) is None: return False, f"the first line names no verdict ({', '.join(VERDICTS)}); a reader cannot tell what it says" + said = verdict_in_first_line(body) + if verdict is not None and said != verdict: + return False, f"the first line says {said}, the VERDICT argument is {verdict}" return True, "ok" @@ -86,14 +95,32 @@ def pending_ids(reviews): def verdicted_at(reviews_and_comments, head): - """Whether any review or issue comment carries this head with a verdict word in it.""" + """Whether any review or issue comment is a verdict about this head: its FIRST line names the head and a verdict. + + Anywhere-in-the-body matching counted a "**Not a verdict** (abc1234 has its READYs)" note as a verdict, and the + post's read-back passed on any older one (review of #2616); the first line is also what pr_review_queue reads.""" for x in reviews_and_comments: - b = x.get("body") or "" - if head[:7] in b and any(v in b for v in VERDICTS): + first = ((x.get("body") or "").strip().splitlines() or [""])[0] + if head[:7] in first and verdict_in_first_line(first) is not None: return True return False +def read_back_ok(obj, head, verdict, kind): + """(ok, reason) for the ONE object a post returned, read again by its id -- pure, for the self-test. + + Proof that THIS post landed, not that some verdict exists: the body read back must classify as the verdict that + was posted, and a review must be attached to the head it is about (`commit_id`).""" + if not obj or "id" not in obj: + return False, f"the {kind} could not be read back by id" + ok, why = classify(obj.get("body") or "", head, verdict) + if not ok: + return False, f"the {kind} read back does not say it: {why}" + if kind == "review" and (obj.get("commit_id") or "") != head: + return False, f"the review is attached to {(obj.get('commit_id') or '?')[:7]}, not the head {head[:7]}" + return True, "ok" + + def gh(*args): p = subprocess.run(["gh", *args], capture_output=True, text=True) return p.returncode, p.stdout, p.stderr @@ -117,7 +144,7 @@ def do_check(repo, pr): pend = pending_ids(reviews) if pend: print(f"PENDING review(s) {pend}: every lane's post to this PR will be rejected until they are cleared") - print(f" clear with: gh api -X DELETE repos/{repo}/pulls/{pr}/reviews/") + print(" post with --as-comment (pr_review_queue reads comments too); the draft belongs to the lane that opened it") print(f"verdicted at this head: {'yes' if verdicted_at(reviews + comments, head) else 'NO'}") return 0 @@ -132,33 +159,41 @@ def do_post(repo, pr, body_file, verdict, dry_run, as_comment=False): except OSError as e: print(f"pr_verdict: {e}", file=sys.stderr) return 2 - ok, why = classify(body, head) + ok, why = classify(body, head, verdict) if not ok: print(f"pr_verdict: refusing to post: {why}", file=sys.stderr) return 2 pend = pending_ids(api_json(f"repos/{repo}/pulls/{pr}/reviews?per_page=100") or []) if pend and not as_comment: print(f"pr_verdict: refusing to post: pending review(s) {pend} block it " - f"(clear: gh api -X DELETE repos/{repo}/pulls/{pr}/reviews/; or pass --as-comment, " - f"which pr_review_queue also reads)", file=sys.stderr) + f"(pass --as-comment, which pr_review_queue also reads; the draft is another lane's work and is " + f"theirs to submit or delete)", file=sys.stderr) return 2 if dry_run: how = "comment" if as_comment else "review" print(f"would post {verdict} about {head[:7]} to {repo}#{pr} as a {how}") return 0 + # Post through the REST API so the result carries the new object's id, and a review is pinned to `head` by + # commit_id (gh pr review attaches to whatever the head is at that moment). + kind = "comment" if as_comment else "review" if as_comment: - rc, out, err = gh("pr", "comment", str(pr), "--repo", repo, "--body-file", body_file) + rc, out, err = gh("api", "-X", "POST", f"repos/{repo}/issues/{pr}/comments", "-F", f"body=@{body_file}") else: - rc, out, err = gh("pr", "review", str(pr), "--repo", repo, "--comment", "--body-file", body_file) + rc, out, err = gh("api", "-X", "POST", f"repos/{repo}/pulls/{pr}/reviews", "-f", "event=COMMENT", + "-f", f"commit_id={head}", "-F", f"body=@{body_file}") if rc != 0: print(f"pr_verdict: post failed: {(err or out).strip()[:200]}", file=sys.stderr) return 2 - back = ((api_json(f"repos/{repo}/pulls/{pr}/reviews?per_page=100") or []) - + (api_json(f"repos/{repo}/issues/{pr}/comments?per_page=100") or [])) - if not verdicted_at(back, head): - print(f"pr_verdict: posted, but no review at {head[:7]} is readable back: check the PR by hand", file=sys.stderr) + try: + oid = json.loads(out).get("id") + except ValueError: + oid = None + path = f"repos/{repo}/issues/comments/{oid}" if as_comment else f"repos/{repo}/pulls/{pr}/reviews/{oid}" + ok, why = read_back_ok(api_json(path) if oid else None, head, verdict, kind) + if not ok: + print(f"pr_verdict: posted, but {why}: check the PR by hand", file=sys.stderr) return 2 - print(f"posted {verdict} about {head[:7]} (read back)") + print(f"posted {verdict} about {head[:7]} as {kind} {oid} (read back by id)") return 0 @@ -187,7 +222,28 @@ def self_test(): assert not verdicted_at([{"body": "abcdef1 looks fine"}], H) # the --as-comment fallback is read back from the ISSUE-comment list, same shape assert verdicted_at([{"body": "abcdef1 READY — posted as a comment"}], H) - print("pr_verdict self-test: ok (head in line 1, stale head, missing verdict, empty, pending, read-back)") + + # review of #2607: NOT READY is its own verdict, never READY + assert verdict_in_first_line("VERDICT at abcdef1: NOT READY") == "NOT READY" + assert verdict_in_first_line("abcdef1 has its READYs") is None + ok, why = classify("VERDICT at abcdef1: NOT READY", H, "READY") + assert not ok and "says NOT READY" in why, why + assert classify("VERDICT at abcdef1: NOT READY", H, "NOT READY") == (True, "ok") + # ... and the VERDICT argument must match the body + ok, why = classify("VERDICT at abcdef1: CHANGES", H, "READY") + assert not ok and "says CHANGES, the VERDICT argument is READY" in why, why + + # review of #2616: a note that mentions the head is not a verdict, and an older verdict is not proof of this post + assert not verdicted_at([{"body": "**Not a verdict** (abcdef1 has its READYs)."}], H) + assert not verdicted_at([{"body": "context first\nVERDICT at abcdef1: READY"}], H) + assert read_back_ok({"id": 7, "body": "VERDICT at abcdef1: READY", "commit_id": H}, H, "READY", "review") == (True, "ok") + assert not read_back_ok(None, H, "READY", "comment")[0] # nothing came back by that id + assert not read_back_ok({"id": 7, "body": "VERDICT at abcdef1: READY", "commit_id": "1234567890"}, H, "READY", + "review")[0] # attached to another head + assert not read_back_ok({"id": 7, "body": "VERDICT at abcdef1: CHANGES"}, H, "READY", "comment")[0] + assert read_back_ok({"id": 8, "body": "VERDICT at abcdef1: READY"}, H, "READY", "comment") == (True, "ok") + print("pr_verdict self-test: ok (head in line 1, stale head, missing verdict, empty, pending, NOT READY, " + "verdict argument, first-line verdicts, read-back by id)") return 0 From 2f9ac8e9f4acdb4ae90acfe41ae544e4428dc586 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 23:25:13 -0400 Subject: [PATCH 2/2] ci: pr_verdict accepts exactly the gate's verdict line (review of #2620) classify() accepted "VERDICT: READY at ", "## VERDICT at : READY" and the sha-first " READY", none of which the merge gate counts (~/lb/verdict_sweep.py STRICT, ~/lb/pr-scan.py). The first line must now match the gate's own regex, `^VERDICT at `?`?:?\s*(READY|CHANGES|NOT READY)`, and the state is taken from it. verdicted_at (--check) mirrors the gate too: any line of a body that is a gate-readable verdict about this head, so a reply that quotes one counts, as it does for the gate. Cross-check against verdict_sweep.py's STRICT on 9 first lines (canonical, backticked, NOT READY, state-before-sha, heading, sha-first, no colon, no space, leading space): 0 disagreements. Mutant (gate check removed): the self-test fails on "VERDICT: READY at abcdef1". Co-Authored-By: Claude Opus 5.5 --- tools/ci/pr_verdict.py | 46 ++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tools/ci/pr_verdict.py b/tools/ci/pr_verdict.py index 7e0f87475..4cbfe0e13 100755 --- a/tools/ci/pr_verdict.py +++ b/tools/ci/pr_verdict.py @@ -48,13 +48,22 @@ import sys HEAD_RE = re.compile(r"\b([0-9a-f]{7,40})\b") +# The merge gate's own reader (~/lb/verdict_sweep.py STRICT, ~/lb/pr-scan.py): `VERDICT at : ` at the start +# of a line, nothing before it -- no `## ` heading, no state before the sha. A body this tool accepts must be one the +# gate counts, or the tool is one more way to post an invisible verdict (review of #2620). +GATE_RE = re.compile(r"^VERDICT at `?([0-9a-f]{7,40})`?:?\s*(READY|CHANGES|NOT READY)") +GATE_RE_LINES = re.compile(GATE_RE.pattern, re.M) # NOT READY first: it contains READY, so a substring test in any other order reads it as READY (review of #2607). VERDICTS = ("NOT READY", "READY", "CHANGES") def first_line_sha(body): - """The head a verdict is about: the 7-hex in the FIRST line, which is what pr_review_queue matches.""" + """The head a verdict is about: the sha of a gate-readable FIRST line (`VERDICT at : ...`), else any 7-40 hex + run in that line (so a malformed line can still be told apart from a stale one).""" first = body.splitlines()[0] if body.strip() else "" + m = GATE_RE.match(first) + if m: + return m.group(1) m = HEAD_RE.search(first) return m.group(1) if m else None @@ -83,7 +92,12 @@ def classify(body, head, verdict=None): return False, f"the first line names {sha}, the head is {head[:7]}: this would review code that is gone" if verdict_in_first_line(body) is None: return False, f"the first line names no verdict ({', '.join(VERDICTS)}); a reader cannot tell what it says" - said = verdict_in_first_line(body) + first = body.splitlines()[0] + g = GATE_RE.match(first) + if g is None: + return False, ("the first line is not `VERDICT at : READY|CHANGES|NOT READY`, the only form the merge gate " + "reads (~/lb/verdict_sweep.py STRICT): no heading, nothing before VERDICT, the state after the sha") + said = g.group(2) if verdict is not None and said != verdict: return False, f"the first line says {said}, the VERDICT argument is {verdict}" return True, "ok" @@ -99,10 +113,12 @@ def verdicted_at(reviews_and_comments, head): Anywhere-in-the-body matching counted a "**Not a verdict** (abc1234 has its READYs)" note as a verdict, and the post's read-back passed on any older one (review of #2616); the first line is also what pr_review_queue reads.""" + # Mirrors the gate: any line of the body that is a gate-readable verdict about this head. (A reply that QUOTES such a + # line counts here exactly as it does for the gate -- which is the reason not to quote one.) for x in reviews_and_comments: - first = ((x.get("body") or "").strip().splitlines() or [""])[0] - if head[:7] in first and verdict_in_first_line(first) is not None: - return True + for m in GATE_RE_LINES.finditer(x.get("body") or ""): + if head.startswith(m.group(1)) or m.group(1).startswith(head[:7]): + return True return False @@ -199,16 +215,16 @@ def do_post(repo, pr, body_file, verdict, dry_run, as_comment=False): def self_test(): H = "abcdef1234567890" - assert first_line_sha("abcdef1 READY — ok\nmore") == "abcdef1" + assert first_line_sha("VERDICT at abcdef1: READY\nmore") == "abcdef1" assert first_line_sha("VERDICT: READY\nabcdef1 in the body") is None # the trap: sha not in line 1 assert verdict_in_first_line("abcdef1 CHANGES — no") == "CHANGES" assert verdict_in_first_line("abcdef1 maybe") is None - ok, why = classify("abcdef1 READY — one holder verified", H) + ok, why = classify("VERDICT at abcdef1: READY", H) assert ok, why ok, why = classify("VERDICT: READY\nthe head is abcdef1", H) assert not ok and "first line names no head" in why, why - ok, why = classify("1234567 READY", H) # stale head + ok, why = classify("VERDICT at 1234567: READY", H) # stale head assert not ok and "code that is gone" in why, why ok, why = classify("abcdef1 approved", H) # no verdict word in line 1 assert not ok and "no verdict" in why, why @@ -217,11 +233,11 @@ def self_test(): assert pending_ids([{"id": 1, "state": "PENDING"}, {"id": 2, "state": "COMMENTED"}]) == [1] assert pending_ids([{"id": 3, "state": "COMMENTED"}]) == [] - assert verdicted_at([{"body": "abcdef1 READY — x"}], H) + assert verdicted_at([{"body": "VERDICT at abcdef1: READY"}], H) assert not verdicted_at([{"body": "READY — no head here"}], H) assert not verdicted_at([{"body": "abcdef1 looks fine"}], H) # the --as-comment fallback is read back from the ISSUE-comment list, same shape - assert verdicted_at([{"body": "abcdef1 READY — posted as a comment"}], H) + assert verdicted_at([{"body": "VERDICT at abcdef1: READY\n(posted as a comment)"}], H) # review of #2607: NOT READY is its own verdict, never READY assert verdict_in_first_line("VERDICT at abcdef1: NOT READY") == "NOT READY" @@ -235,7 +251,15 @@ def self_test(): # review of #2616: a note that mentions the head is not a verdict, and an older verdict is not proof of this post assert not verdicted_at([{"body": "**Not a verdict** (abcdef1 has its READYs)."}], H) - assert not verdicted_at([{"body": "context first\nVERDICT at abcdef1: READY"}], H) + # the gate counts a verdict line anywhere in the body, so --check does too (and so a quoted one is a vote) + assert verdicted_at([{"body": "context first\nVERDICT at abcdef1: READY"}], H) + + # review of #2620: only the gate's form is accepted -- the state after `at `, nothing before VERDICT + for bad in ("VERDICT: READY at abcdef1", "## VERDICT at abcdef1: READY", "abcdef1 READY — ok", "Verdict at abcdef1: READY"): + ok, why = classify(bad, H, "READY") + assert not ok, (bad, why) + assert classify("VERDICT at `abcdef1`: READY — fine", H, "READY") == (True, "ok") + assert not verdicted_at([{"body": "VERDICT: READY at abcdef1"}], H) # the gate cannot see it assert read_back_ok({"id": 7, "body": "VERDICT at abcdef1: READY", "commit_id": H}, H, "READY", "review") == (True, "ok") assert not read_back_ok(None, H, "READY", "comment")[0] # nothing came back by that id assert not read_back_ok({"id": 7, "body": "VERDICT at abcdef1: READY", "commit_id": "1234567890"}, H, "READY",