Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmake/tests-gpu-hold.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,7 @@ add_test(NAME gpu_hold_hold_dir COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_
# for 40 s inside another lane's 45-minute hold. gpu-hold exports the exact token line it wrote (GPU_HOLD_TICKET) and the
# label, and this asserts the child sees both, on a sandboxed HOME.
add_test(NAME gpu_hold_child_identity COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_child_identity_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold)

# docs/ci-runners.md's exit-code table is a contract (review of engine#2539): every "<script> exits/sets it" in a row
# must be backed by `exit <rc>` or `rc=<rc>` in that script, and every row must name an emitter.
add_test(NAME exit_code_table COMMAND python3 -I ${CMAKE_SOURCE_DIR}/tests/exit_code_table_test.py ${CMAKE_SOURCE_DIR})
2 changes: 1 addition & 1 deletion docs/ci-runners.md
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ Each script's own header stays the reference for its edge cases.
|---|---|---|---|
| 0 | the job (or gate) passed / the window closed on its own | all | - |
| 1 | the job's own failure: a check failed | hw-gate.sh, the payload | no: read the log |
| 2 | one code, four causes; **the stderr line says which** and is part of the contract: (a) a **usage error** (`unknown option … (try -h)`, `no command given`): the command line is wrong, not the box; (b) **could not run / INCOMPLETE**: an input was absent (a model, text, a binary, a checkout) or a build or memory floor refused it. hw-gate reports 2 even when some rows failed, because an untested input means the run did not answer its question; (c) the box could not set up the ticket (`cannot create <queue>`, `cannot write <ticket>`); (d) a **policy refusal** (npu-hold: a shared queue while npu-serve-keeper reads it) | gpu-hold and hw-gate.sh exit it; npu-hold exits (d) | **read stderr first**. (a) and (d): fix the command or config, because re-queueing repeats the same mistake; (b): after providing the input; (c): after fixing the box |
| 2 | one code, four causes; **the stderr line says which** and is part of the contract (each phrase below is what the script prints, so grep your log for it): (a) a **usage error**, `gpu-hold: unknown option` or `gpu-hold: no command given`: the command line is wrong, not the box; (b) **could not run / INCOMPLETE**, `INCOMPLETE (an input was absent: nothing was tested)`: an input was absent (a model, text, a binary, a checkout) or a build or memory floor refused it. hw-gate reports 2 even when some rows failed, because an untested input means the run did not answer its question; (c) the box could not set up the ticket, `gpu-hold: cannot create` or `gpu-hold: cannot write`; (d) a **policy refusal**, `npu-hold: REFUSED:` (a shared queue while npu-serve-keeper reads it) | gpu-hold and hw-gate.sh exit it; npu-hold exits (d) | **read stderr first**. (a) and (d): fix the command or config, because re-queueing repeats the same mistake; (b): after providing the input; (c): after fixing the box |
| 3 | the server under test never came up | hw-gate.sh exits it from its server-start step | read the server log |
| 69 | the job ran, but coder-llm did not answer `:8090` after the window | window-run.sh (computed by `final_rc`, not a literal exit) | see "The PR-Agent model runs in gpu-hold windows" |
| 75 | the queue never came round within `-w` | gpu-hold exits it; npu-hold and coder-llm-ticketed pass it through | yes, but **at the back** of the queue. That is why coder-llm queues with a 24 h `-w`, and why a lane should size `-w` for the queue instead of treating 75 as a free retry |
Expand Down
97 changes: 97 additions & 0 deletions tests/exit_code_table_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# tests/exit_code_table_test.py [ROOT] — docs/ci-runners.md's "Exit codes on the GPU and NPU path" table against the
# scripts it names (review of engine#2539: the table is a contract and prose rots). Every "<script> exits it" or
# "<script> sets it" in a row's scripts cell must be backed by `exit <rc>` or `rc=<rc>` in that script, and every row
# must name an emitter (a script that exits, sets or passes the code, or "any payload" / "all" / "the payload").
# Each verb is checked against the script names written since the previous verb, so "A exits it, B exits it" checks
# both (review of #2546). A code that is computed rather than literal is written that way in the row ("<script>
# (computed by ...)"): the named script must contain the compute site (`echo <rc>`), so the name itself is checked.
# The 2 row's quoted stderr phrases (review of engine#2544: a lane greps its log for them) must each be printed by one of
# the scripts, verbatim.
import argparse
import re
import sys
from pathlib import Path

ap = argparse.ArgumentParser(description="Check docs/ci-runners.md's exit-code table against the scripts it names.")
ap.add_argument("root", nargs="?", default=str(Path(__file__).resolve().parent.parent),
help="engine checkout to check (default: this test's own tree)")
root = Path(ap.parse_args().root)
SCRIPTS = {"gpu-hold": "scripts/gpu-hold", "npu-hold": "scripts/npu-hold", "hw-gate.sh": "scripts/hw-gate.sh",
"window-run.sh": "scripts/window-run.sh", "coder-llm-ticketed": "scripts/coder-llm-ticketed"}

doc = (root / "docs/ci-runners.md").read_text()
m = re.search(r"^## Exit codes on the GPU and NPU path.*?(?=^## )", doc, re.S | re.M)
if not m:
print("FAIL: no 'Exit codes on the GPU and NPU path' section in docs/ci-runners.md")
sys.exit(1)
rows = [l for l in m.group(0).splitlines() if re.match(r"^\| \**\d+\** \|", l)]
fail, checked = 0, 0
if len(rows) < 10:
print(f"FAIL: the table has {len(rows)} rows, expected at least 10 (did the row format change?)")
fail = 1
for row in rows:
cells = [c.strip() for c in row.strip().strip("|").split("|")]
rc, where = cells[0].strip("*"), cells[2]
named = False
def names(text):
return [n for n in SCRIPTS if re.search(r"(?<![\w-])" + re.escape(n) + r"(?![\w-])", text)]
for seg in where.split(";"):
if re.search(r"pass(es)? it through|relabels|any payload|^all$|the payload", seg):
named = True
comp = re.search(r"\(computed by", seg)
if comp: # computed at run time (window-run's 69): no literal exit, so check the named script computes it
who = names(seg[:comp.start()])
if not who:
print(f"FAIL: rc {rc}: 'computed by' names no script ({seg.strip()[:80]})")
fail = 1
for name in who:
named = True
checked += 1
if not re.search(rf"\becho {rc}\b|\bexit {rc}\b|\brc={rc}\b", (root / SCRIPTS[name]).read_text()):
print(f"FAIL: rc {rc}: the table says {name} computes it, but {SCRIPTS[name]} has no 'echo {rc}', 'exit {rc}' or 'rc={rc}'")
fail = 1
continue
verbs = list(re.finditer(r"\b(exits?|sets) (it\b|\()", seg))
for k, verb in enumerate(verbs):
named = True
for name in names(seg[verbs[k - 1].end() if k else 0:verb.start()]):
checked += 1
if not re.search(rf"\bexit {rc}\b|\brc={rc}\b", (root / SCRIPTS[name]).read_text()):
print(f"FAIL: rc {rc}: the table says {name} {verb.group(1)} it, but {SCRIPTS[name]} has no 'exit {rc}' or 'rc={rc}'")
fail = 1
if not named:
print(f"FAIL: rc {rc}: the scripts cell names no emitter ({where[:80]})")
fail = 1
two = [r for r in rows if re.match(r"^\| \**2\** \|", r)]
phrases = re.findall(r"`([^`]+)`", two[0].split("|")[2]) if two else []
if len(phrases) < 6:
print(f"FAIL: the 2 row quotes {len(phrases)} stderr phrases, expected at least 6 (one per cause)")
fail = 1
alltext = "".join((root / f).read_text() for f in SCRIPTS.values())
for ph in phrases:
if ph not in alltext:
print(f"FAIL: rc 2: the stderr phrase '{ph}' is not printed by any of the scripts")
fail = 1
FLOOR = 10 # the claims on main when this was written: a parser that stops finding claims must not pass as "nothing to check"
if checked < FLOOR:
print(f"FAIL: only {checked} (code, script) claims were checked, the table has at least {FLOOR}; the parser no longer reads it")
fail = 1
if not fail:
print(f"exit_code_table: ok ({len(rows)} rows, {checked} (code, script) claims backed by the scripts; {len(phrases)} rc 2 stderr phrases found verbatim)")
sys.exit(fail)
Loading