Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions scripts/hw-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ trap 'exit 143' TERM INT HUP # so a killed run (timeout, CI cancel) still rele
# (engine#1504; tests/hw_gate_mem_test.sh).
gpu() {
if ! take_token; then
result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
exit 2
fi
wait_coder_idle
Expand All @@ -414,7 +414,7 @@ gpu() {
release_token
sleep "${HW_GATE_MEM_RETRY_SLEEP:-60}"
if ! take_token; then
result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
exit 2
fi
done
Expand Down Expand Up @@ -513,7 +513,7 @@ if [ -z "$BIN" ]; then
# reading), and the compile runs under mem-guard --term, which ends the whole thermal-run scope below
# MEM_FLOOR GB. A guard stop is not heat: the loop below does not resume it.
if ! "$ROOT/scripts/mem-wait.sh" "$BUILD_MEM_FLOOR" "$BUILD_MEM_WAIT" >> "$OUT/build.log" 2>&1; then
result build FAIL $(( $(date +%s) - t0 )) "not started: $(tail -n 1 "$OUT/build.log")"
result build SKIP $(( $(date +%s) - t0 )) "not started (could not run): $(tail -n 1 "$OUT/build.log")"
exit 2
fi
# thermal-run kills a build at 93 C (the box is shared); ninja resumes where it stopped
Expand Down Expand Up @@ -543,7 +543,7 @@ if [ -z "$BIN" ]; then
result build PASS $(( $(date +%s) - t0 )) "$build"
else
if [ $guarded = 1 ]; then
result build FAIL $(( $(date +%s) - t0 )) "stopped by the memory guard: $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)"
result build SKIP $(( $(date +%s) - t0 )) "stopped by the memory guard (could not run): $(grep "^GUARD:" "$OUT/build.log" | tail -n 1); re-run once memory frees (engine#1504)"
exit 2
fi
result build FAIL $(( $(date +%s) - t0 )) "see $OUT/build.log"
Expand All @@ -566,7 +566,7 @@ fi
# ---- the GPU token, before the first GPU step (gpu() would take it too, but its output is a step log)
if has ops || has kat || has needle || has kld || has placement; then
if ! take_token; then
result gpu-token FAIL "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
result gpu-token SKIP "$TOKEN_WAIT" "$TOKEN held by another job for ${TOKEN_WAIT}s: $(cat "$TOKEN" 2> /dev/null)"
exit 2
fi
fi
Expand Down
20 changes: 20 additions & 0 deletions tests/hw_gate_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ def kat(url, *args):
return p.returncode, p.stdout


def result_build_real_failure_missing(src):
"""The one build FAIL that must survive: a compile that ran and failed, not a box condition."""
return 'result build FAIL $(( $(date +%s) - t0 )) "see ' not in src


def script_invariants():
"""The gate's lock placement, from the script itself (no GPU needed): engine#682 took ~/npu.lock off the
GPU steps, because an NPU serve and the GPU run concurrently here (engine#673 measured ~87 % of the NPU
Expand Down Expand Up @@ -117,6 +122,21 @@ def script_invariants():
bad.append(f"{absent} must be SKIP, not FAIL: an absent input is not a result (engine#2501)")
if "infra=1" not in line: # SKIP alone would let "ran nothing" look like "all passed"
bad.append(f"{absent} must keep infra=1, so a run with no inputs still exits 2")
# engine#2501/#2522's rule, third site: a GPU token held by another job is "could not run", not a
# verdict on the PR under test. As a FAIL row it reds every PR for a queue condition -- the most
# frequently hit of the three infra-as-failure sites, because any lane holding the token causes it.
if 'result gpu-token FAIL' in src:
bad.append("a held GPU token is an infra condition: the row must be SKIP, not FAIL")
if 'result gpu-token SKIP' not in src:
bad.append("the gpu-token row must exist and be SKIP (it was renamed or removed)")
# Same rule, third and fourth sites: a build that never started, or that the memory guard stopped,
# is "could not run". The compile failure stays FAIL -- that one IS the job's verdict.
if 'result build FAIL' in src and '"not started' in src.split('result build FAIL', 1)[1][:40]:
bad.append("a build that never started is an infra condition: the row must be SKIP, not FAIL")
if 'stopped by the memory guard' in src and 'result build SKIP' not in src:
bad.append("a memory-guard stop is an infra condition: the row must be SKIP, not FAIL")
if result_build_real_failure_missing(src):
bad.append("the compile-failure build row must stay FAIL: that is the job's own verdict")
if src.find('result "kld $name" FAIL') < 0:
bad.append("the KLD mismatch row must stay FAIL: a verify mismatch is a result, not an absent input")
return bad
Expand Down
Loading