diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index f77b5e491..639866cdc 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -392,7 +392,7 @@ trap 'exit 143' TERM INT HUP # so a killed run (timeout, CI cancel) still rele # (engine#1504; tests/hw_gate_mem_test.sh). gpu() { if ! take_token; then - result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" exit 2 fi wait_coder_idle @@ -414,7 +414,7 @@ gpu() { release_token sleep "${HW_GATE_MEM_RETRY_SLEEP:-60}" if ! take_token; then - result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" exit 2 fi done @@ -513,7 +513,7 @@ if [ -z "$BIN" ]; then # reading), and the compile runs under mem-guard --term, which ends the whole thermal-run scope below # MEM_FLOOR GB. A guard stop is not heat: the loop below does not resume it. if ! "$ROOT/scripts/mem-wait.sh" "$BUILD_MEM_FLOOR" "$BUILD_MEM_WAIT" >> "$OUT/build.log" 2>&1; then - result build FAIL $(( $(date +%s) - t0 )) "not started: $(tail -n 1 "$OUT/build.log")" + result build SKIP $(( $(date +%s) - t0 )) "not started (could not run): $(tail -n 1 "$OUT/build.log")" exit 2 fi # thermal-run kills a build at 93 C (the box is shared); ninja resumes where it stopped @@ -543,7 +543,7 @@ if [ -z "$BIN" ]; then result build PASS $(( $(date +%s) - t0 )) "$build" else if [ $guarded = 1 ]; then - result build FAIL $(( $(date +%s) - t0 )) "stopped by the memory guard: $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)" + result build SKIP $(( $(date +%s) - t0 )) "stopped by the memory guard (could not run): $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)" exit 2 fi result build FAIL $(( $(date +%s) - t0 )) "see $OUT/build.log" @@ -566,7 +566,7 @@ fi # ---- the GPU token, before the first GPU step (gpu() would take it too, but its output is a step log) if has ops || has kat || has needle || has kld || has placement; then if ! take_token; then - result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" + result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)" exit 2 fi fi diff --git a/tests/hw_gate_test.py b/tests/hw_gate_test.py index acf63d7e7..2bcb06631 100644 --- a/tests/hw_gate_test.py +++ b/tests/hw_gate_test.py @@ -88,6 +88,11 @@ def kat(url, *args): return p.returncode, p.stdout +def result_build_real_failure_missing(src): + """The one build FAIL that must survive: a compile that ran and failed, not a box condition.""" + return 'result build FAIL $(( $(date +%s) - t0 )) "see ' not in src + + def script_invariants(): """The gate's lock placement, from the script itself (no GPU needed): engine#682 took ~/npu.lock off the GPU steps, because an NPU serve and the GPU run concurrently here (engine#673 measured ~87 % of the NPU @@ -117,6 +122,21 @@ def script_invariants(): bad.append(f"{absent} must be SKIP, not FAIL: an absent input is not a result (engine#2501)") if "infra=1" not in line: # SKIP alone would let "ran nothing" look like "all passed" bad.append(f"{absent} must keep infra=1, so a run with no inputs still exits 2") + # engine#2501/#2522's rule, third site: a GPU token held by another job is "could not run", not a + # verdict on the PR under test. As a FAIL row it reds every PR for a queue condition -- the most + # frequently hit of the three infra-as-failure sites, because any lane holding the token causes it. + if 'result gpu-token FAIL' in src: + bad.append("a held GPU token is an infra condition: the row must be SKIP, not FAIL") + if 'result gpu-token SKIP' not in src: + bad.append("the gpu-token row must exist and be SKIP (it was renamed or removed)") + # Same rule, third and fourth sites: a build that never started, or that the memory guard stopped, + # is "could not run". The compile failure stays FAIL -- that one IS the job's verdict. + if 'result build FAIL' in src and '"not started' in src.split('result build FAIL', 1)[1][:40]: + bad.append("a build that never started is an infra condition: the row must be SKIP, not FAIL") + if 'stopped by the memory guard' in src and 'result build SKIP' not in src: + bad.append("a memory-guard stop is an infra condition: the row must be SKIP, not FAIL") + if result_build_real_failure_missing(src): + bad.append("the compile-failure build row must stay FAIL: that is the job's own verdict") if src.find('result "kld $name" FAIL') < 0: bad.append("the KLD mismatch row must stay FAIL: a verify mismatch is a result, not an absent input") return bad