From f66cfbe11b40314593d4846435ddd40acf394c69 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 22:01:01 -0400 Subject: [PATCH 1/2] tools: pr_check_verdict.py says which red a PR's red is (engine#2077) #2077 measured it: pr_agent ran 32 of the last 40 pull_request runs to CANCELLED, because every push cancels the queued review, and the fleet's merge gates read that as a check that failed. The issue's own closing advice is the spec for this tool: "the first thing to read from now on is the check's conclusion, not its colour: CANCELLED and FAILURE need different answers." Four classes, fed only by the conclusion: FAILED ran and failed -- a verdict about the diff CANCELLED did not run -- says nothing about the diff SKIPPED deliberately not run (a gate that does not apply) neutral SUCCESS, or a run still in progress An unrecognised conclusion is FAILED, never neutral: a word this tool does not know must not be read as a pass. And the exit code keeps the three outcomes apart -- 0 yes, 1 no, 3 unknown -- because a caller that folds "did not run" into "failed" reproduces the exact defect (#2077: 32 cancelled reviews read as 32 rejections). Ten self-test arms, no network: one green, one failure, one cancelled (unknown, not no), failure outranking cancelled, a deliberate skip being fine, still-running, a timeout as a real failure, an unknown conclusion as a failure, no checks at all, and the issue's own requirement stated as its own arm. Checked against the shape #2077 quotes for #2007 (build SUCCESS, pr_agent CANCELLED, hw_gate SKIPPED): exit 3, "mergeable: unknown (0 failed, 1 cancelled, 1 skipped, 1 ok)". Refs #2077. --- CMakeLists.txt | 1 + tools/pr_check_verdict.py | 162 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100755 tools/pr_check_verdict.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f60faae3..54f0d7a91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,6 +201,7 @@ add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.s # is not AMD's. Local git repositories only: no network, no GPU. add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh) add_test(NAME bump_hrx_report COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_report_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-report.sh) +add_python_test(pr_check_verdict_self ${CMAKE_SOURCE_DIR}/tools/pr_check_verdict.py --self-test) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) diff --git a/tools/pr_check_verdict.py b/tools/pr_check_verdict.py new file mode 100755 index 000000000..ee3f7a255 --- /dev/null +++ b/tools/pr_check_verdict.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""pr_check_verdict.py -- say what a PR's "red" actually is, per check, from its conclusion. + +Why this exists (engine#2077): `pr_agent` read red on 32 of 40 pull_request runs, and the conclusion +was CANCELLED -- every push cancels the queued review, so a check that never ran was being read as a +check that failed. The issue's own closing advice: "the first thing to read from now on is the check's +conclusion, not its colour: CANCELLED and FAILURE need different answers." This is that reading, made +mechanical, so a lane does not have to remember which red is which. + +A conclusion is classified four ways, and the fourth is the point: + + FAILED -- ran and failed. A verdict about the diff: fix it. + CANCELLED -- did not run. Says nothing about the diff; re-run (or wait for the newer run). + SKIPPED -- deliberately not run (a gate that does not apply). Says nothing about the diff. + neutral -- SUCCESS, or a run still in progress (its conclusion is empty). + +`mergeable` is yes only when every check is neutral. `unknown` is reported distinctly from `no`, because +"a check was cancelled" and "a check failed" must not both print as "no" -- that conflation is what made +32 cancelled reviews look like 32 rejections. The exit code keeps the three apart too: 0 yes, 1 no, +3 unknown, so a caller cannot silently fold "did not run" into "failed". + +usage: pr_check_verdict.py classify a saved `statusCheckRollup` document + pr_check_verdict.py --self-test fixtures only, no network +""" +import json +import sys + +FAILED = "FAILED" +CANCELLED = "CANCELLED" +SKIPPED = "SKIPPED" +NEUTRAL = "neutral" + +CLASS_OF = { + "FAILURE": FAILED, + "TIMED_OUT": FAILED, + "STARTUP_FAILURE": FAILED, + "CANCELLED": CANCELLED, + "SKIPPED": SKIPPED, + "NEUTRAL": NEUTRAL, + "SUCCESS": NEUTRAL, + "": NEUTRAL, # still running: no conclusion yet, so nothing to say about the diff +} + + +def classify(conclusion): + """A conclusion string -> one of the four classes. Unknown conclusions are FAILED, never neutral: + a conclusion this tool does not know must not be read as a pass.""" + return CLASS_OF.get((conclusion or "").upper(), FAILED if conclusion else NEUTRAL) + + +def read_rollup(doc): + """[(name, status, conclusion, class)] from a statusCheckRollup document (CheckRun and StatusContext).""" + rows = [] + for c in doc or []: + name = c.get("name") or c.get("context") or "?" + if c.get("__typename") == "StatusContext": + rows.append((name, c.get("state", ""), c.get("state", ""), classify(c.get("state")))) + else: + rows.append((name, c.get("status", ""), c.get("conclusion") or "", classify(c.get("conclusion")))) + return rows + + +def verdict(rows): + """(mergeable, counts) where mergeable is 'yes', 'no', or 'unknown'.""" + counts = {FAILED: 0, CANCELLED: 0, SKIPPED: 0, NEUTRAL: 0} + for _, _, _, cls in rows: + counts[cls] += 1 + if counts[FAILED]: + return "no", counts + if counts[CANCELLED]: + return "unknown", counts + return "yes", counts + + +def report(rows): + out = [] + for name, status, conclusion, cls in rows: + if cls == NEUTRAL and status.upper() == "IN_PROGRESS": + out.append(f" {cls:<9} {name} (still running)") + elif cls != NEUTRAL: + out.append(f" {cls:<9} {name} (conclusion {conclusion or 'none'})") + return "\n".join(out) + + +def self_test(): + cases = [ + # (rollup, expected verdict, why) + ([{"name": "build", "status": "COMPLETED", "conclusion": "SUCCESS"}], "yes", "one green check"), + ([{"name": "build", "status": "COMPLETED", "conclusion": "FAILURE"}], "no", "a failure is a no"), + ([{"name": "pr_agent", "status": "COMPLETED", "conclusion": "CANCELLED"}], "unknown", + "a cancelled check never ran -- not a verdict about the diff"), + ([{"name": "a", "status": "COMPLETED", "conclusion": "CANCELLED"}, + {"name": "b", "status": "COMPLETED", "conclusion": "FAILURE"}], "no", "failure outranks cancelled"), + ([{"name": "hw_gate", "status": "COMPLETED", "conclusion": "SKIPPED"}], "yes", "a deliberate skip is not a red"), + ([{"name": "x", "status": "IN_PROGRESS", "conclusion": None}], "yes", "running with no conclusion yet"), + ([{"name": "x", "status": "COMPLETED", "conclusion": "TIMED_OUT"}], "no", "a timeout is a real failure"), + ([{"name": "x", "status": "COMPLETED", "conclusion": "SOMETHING_NEW"}], "no", + "an unknown conclusion must not read as a pass"), + ([], "yes", "no checks at all"), + ] + bad = 0 + for doc, want, why in cases: + got, _ = verdict(read_rollup(doc)) + if got != want: + print(f"FAIL self-test: {why}: got {got}, wanted {want}") + bad += 1 + else: + print(f"ok {why}") + # the classification the issue asks for, stated as its own arm + if classify("CANCELLED") == CANCELLED and classify("FAILURE") == FAILED: + print("ok CANCELLED and FAILURE are different classes (engine#2077)") + else: + print("FAIL CANCELLED and FAILURE must be different classes") + bad += 1 + print(("ok " if not bad else "FAIL ") + f"{len(cases) + 1} arms, {bad} failure(s)") + return 1 if bad else 0 + + +def main(argv): + if len(argv) == 2 and argv[1] == "--self-test": + return self_test() + if len(argv) != 2: + print(__doc__.strip(), file=sys.stderr) + return 2 + try: + with open(argv[1], encoding="utf-8") as f: + doc = json.load(f) + except OSError as e: + print(f"pr_check_verdict: cannot read {argv[1]}: {e}", file=sys.stderr) + return 2 + except json.JSONDecodeError as e: + print(f"pr_check_verdict: {argv[1]} is not JSON: {e}", file=sys.stderr) + return 2 + rows = read_rollup(doc) + v, counts = verdict(rows) + body = report(rows) + if body: + print(body) + print(f"mergeable: {v} ({counts[FAILED]} failed, {counts[CANCELLED]} cancelled, " + f"{counts[SKIPPED]} skipped, {counts[NEUTRAL]} ok)") + # Three exit codes, not two: a caller that treats "unknown" as "no" is the defect this tool exists to + # remove (engine#2077 -- 32 cancelled reviews read as 32 rejections), so the shell must be able to tell + # "a check failed" from "a check did not run". + return {"yes": 0, "no": 1, "unknown": 3}[v] + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 30b2b497aa85ddf58c7ab13969a9fc55e489bd3b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 22:20:58 -0400 Subject: [PATCH 2/2] CMakeLists: register pr_check_verdict_self after add_python_test is defined, not before --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54f0d7a91..275a3f277 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,7 +201,6 @@ add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.s # is not AMD's. Local git repositories only: no network, no GPU. add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh) add_test(NAME bump_hrx_report COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_report_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-report.sh) -add_python_test(pr_check_verdict_self ${CMAKE_SOURCE_DIR}/tools/pr_check_verdict.py --self-test) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) @@ -307,6 +306,7 @@ add_python_test(python_test_order_check ${CMAKE_SOURCE_DIR}/tests/python_test_or add_python_test(python_test_order_check_self ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --self-test) add_python_test(ci_targets ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +add_python_test(pr_check_verdict_self ${CMAKE_SOURCE_DIR}/tools/pr_check_verdict.py --self-test) # engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail. # `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and # tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink.