From 793bdefd32101676609e0ba4ed6a1bb0b51dcf43 Mon Sep 17 00:00:00 2001 From: pi-prs Date: Sun, 15 Nov 2026 11:18:26 -0400 Subject: [PATCH] =?UTF-8?q?hw-gate:=20infra=20rows=20are=20SKIP,=20not=20F?= =?UTF-8?q?AIL=20=E2=80=94=20gpu-token=20(3=20sites)=20and=20build=20(2=20?= =?UTF-8?q?of=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine#2522 made an ABSENT INPUT a SKIP row and left a KLD verify mismatch as FAIL, on the rule that a row says what happened to the RUN, not who to blame for the box. Two more sites were still infra-as-FAIL, and the token one is the most frequently hit in the fleet: gpu-token FAIL 300s ~/GPU-TOKEN held by another job for 300s: gpu-hold coder-llm pid 1027021 Any lane holding the token reds every PR for a queue condition no PR can fix -- exactly the harm #2501 described, one row over. The same for a build that never started, and for a build the memory guard stopped. All three exit 2 already, so this changes only the ROW: result gpu-token FAIL ... -> SKIP (3 sites: :395 :417 :569) result build FAIL ... "not started: ..." -> SKIP "not started (could not run): ..." result build FAIL ... "stopped by the memory guard: ..." -> SKIP "stopped by the memory guard ..." NOT changed: `result build FAIL ... "see $OUT/build.log"`. A compile that ran and failed IS the job's verdict and must stay a failure. The npu/npuserve rows were already SKIP, so the NPU path needed nothing. script_invariants() gains four assertions, beside the absent-input ones from #2522: - no `result gpu-token FAIL`; `result gpu-token SKIP` still exists - no build row whose detail begins "not started"; the memory-guard row is SKIP - the compile-failure build row still EXISTS and is still FAIL Mutation-checked four ways, all caught: one gpu-token site back to FAIL -> "a held GPU token is an infra condition ..." gpu-token row renamed away -> "the gpu-token row must exist and be SKIP ..." build "not started" back to FAIL -> "a build that never started is an infra condition ..." compile failure flipped to SKIP -> "the compile-failure build row must stay FAIL ..." both correct -> hw_gate_test: PASS Measured, not read: a gate run with HW_GATE_GPU_HOLD=0 --phases placement blocked on the token held by `gpu-hold coder-llm pid 1027021` and wrote no row before a 240 s timeout, because take_token() is called after the wrapper that flag skips. The token was not bypassed and no GPU job ran. --- scripts/hw-gate.sh | 10 +++++----- tests/hw_gate_test.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index f77b5e491..639866cdc 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -392,7 +392,7 @@ trap 'exit 143' TERM INT HUP # so a killed run (timeout, CI cancel) still rele # (engine#1504; tests/hw_gate_mem_test.sh). gpu() { if ! take_token; then - result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" exit 2 fi wait_coder_idle @@ -414,7 +414,7 @@ gpu() { release_token sleep "${HW_GATE_MEM_RETRY_SLEEP:-60}" if ! take_token; then - result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" exit 2 fi done @@ -513,7 +513,7 @@ if [ -z "$BIN" ]; then # reading), and the compile runs under mem-guard --term, which ends the whole thermal-run scope below # MEM_FLOOR GB. A guard stop is not heat: the loop below does not resume it. if ! "$ROOT/scripts/mem-wait.sh" "$BUILD_MEM_FLOOR" "$BUILD_MEM_WAIT" >> "$OUT/build.log" 2>&1; then - result build FAIL $(( $(date +%s) - t0 )) "not started: $(tail -n 1 "$OUT/build.log")" + result build SKIP $(( $(date +%s) - t0 )) "not started (could not run): $(tail -n 1 "$OUT/build.log")" exit 2 fi # thermal-run kills a build at 93 C (the box is shared); ninja resumes where it stopped @@ -543,7 +543,7 @@ if [ -z "$BIN" ]; then result build PASS $(( $(date +%s) - t0 )) "$build" else if [ $guarded = 1 ]; then - result build FAIL $(( $(date +%s) - t0 )) "stopped by the memory guard: $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)" + result build SKIP $(( $(date +%s) - t0 )) "stopped by the memory guard (could not run): $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)" exit 2 fi result build FAIL $(( $(date +%s) - t0 )) "see $OUT/build.log" @@ -566,7 +566,7 @@ fi # ---- the GPU token, before the first GPU step (gpu() would take it too, but its output is a step log) if has ops || has kat || has needle || has kld || has placement; then if ! take_token; then - result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" exit 2 fi fi diff --git a/tests/hw_gate_test.py b/tests/hw_gate_test.py index acf63d7e7..2bcb06631 100644 --- a/tests/hw_gate_test.py +++ b/tests/hw_gate_test.py @@ -88,6 +88,11 @@ def kat(url, *args): return p.returncode, p.stdout +def result_build_real_failure_missing(src): + """The one build FAIL that must survive: a compile that ran and failed, not a box condition.""" + return 'result build FAIL $(( $(date +%s) - t0 )) "see ' not in src + + def script_invariants(): """The gate's lock placement, from the script itself (no GPU needed): engine#682 took ~/npu.lock off the GPU steps, because an NPU serve and the GPU run concurrently here (engine#673 measured ~87 % of the NPU @@ -117,6 +122,21 @@ def script_invariants(): bad.append(f"{absent} must be SKIP, not FAIL: an absent input is not a result (engine#2501)") if "infra=1" not in line: # SKIP alone would let "ran nothing" look like "all passed" bad.append(f"{absent} must keep infra=1, so a run with no inputs still exits 2") + # engine#2501/#2522's rule, third site: a GPU token held by another job is "could not run", not a + # verdict on the PR under test. As a FAIL row it reds every PR for a queue condition -- the most + # frequently hit of the three infra-as-failure sites, because any lane holding the token causes it. + if 'result gpu-token FAIL' in src: + bad.append("a held GPU token is an infra condition: the row must be SKIP, not FAIL") + if 'result gpu-token SKIP' not in src: + bad.append("the gpu-token row must exist and be SKIP (it was renamed or removed)") + # Same rule, third and fourth sites: a build that never started, or that the memory guard stopped, + # is "could not run". The compile failure stays FAIL -- that one IS the job's verdict. + if 'result build FAIL' in src and '"not started' in src.split('result build FAIL', 1)[1][:40]: + bad.append("a build that never started is an infra condition: the row must be SKIP, not FAIL") + if 'stopped by the memory guard' in src and 'result build SKIP' not in src: + bad.append("a memory-guard stop is an infra condition: the row must be SKIP, not FAIL") + if result_build_real_failure_missing(src): + bad.append("the compile-failure build row must stay FAIL: that is the job's own verdict") if src.find('result "kld $name" FAIL') < 0: bad.append("the KLD mismatch row must stay FAIL: a verify mismatch is a result, not an absent input") return bad