diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 51c46f447..310a5e036 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -635,7 +635,14 @@ if has kat || has needle || has kld; then while IFS='|' read -r name file repeat donee srvargs; do want_model "$name" || continue model=$MODELS/$file base=$STORE/kld/$name.kld - if [ ! -f "$model" ]; then result "model $name" FAIL 0 "model missing: $model"; infra=1; continue; fi + # engine#2501: a model file that is not on this runner is an ENVIRONMENT fact, not a gate result, and a FAIL + # row here reds every PR's gate for a reason no PR can fix. SKIP it and name the path -- and collect it, so the + # run ends with one line saying which models were not exercised instead of a silently smaller matrix. + if [ ! -f "$model" ]; then + result "model $name" SKIP 0 "not on this runner: $model" + missing_models="${missing_models:-} $name" + continue + fi dokat=0; has kat && dokat=1 needle=0; [ "$donee" = 1 ] && has needle && needle=1 dokld=0; has kld && [ -f "$TEXT" ] && [ -z "$srvargs" ] && dokld=1 @@ -802,6 +809,10 @@ rings=$(ring_timeouts) [ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$OUT/summary.md" >> "$GITHUB_STEP_SUMMARY" cat "$OUT/summary.md" if [ "$nfail" != 0 ]; then + # engine#2501: coverage that did not happen has to be visible on the run, or the matrix looks complete. + if [ -n "${missing_models:-}" ]; then + note "::warning::model rows skipped because the GGUF is not on this runner:"$missing_models + fi [ "$infra" = 1 ] && exit 2 exit 1 fi diff --git a/tests/hw_gate_test.py b/tests/hw_gate_test.py index 4609ad87b..a67f4d840 100644 --- a/tests/hw_gate_test.py +++ b/tests/hw_gate_test.py @@ -88,6 +88,29 @@ def kat(url, *args): return p.returncode, p.stdout +def missing_model_is_a_skip(): + """engine#2501: a GGUF that is not on this runner must SKIP its row and be named at the end, not FAIL it. + + The FAIL cost every PR's gate a row for a reason no PR could fix, and the row was red on main's own pin. The + opposite mistake is worse: a silently skipped model makes the matrix look complete, so the run has to say which + rows it did not exercise. Both halves are asserted from the script text, like the lock invariant above (no GPU, + no models: the behaviour under test is what the missing-file branch does).""" + bad = [] + with open(os.path.join(ROOT, "scripts", "hw-gate.sh")) as f: + src = f.read() + if 'result "model $name" FAIL 0 "model missing' in src: + bad.append("a missing GGUF still FAILs its row (engine#2501)") + if 'result "model $name" SKIP 0 "not on this runner' not in src: + bad.append("a missing GGUF does not SKIP its row with the path named") + if "missing_models=" not in src or "model rows skipped because the GGUF is not on this runner" not in src: + bad.append("skipped model rows are not named in the run summary") + # the skip must not set infra: a missing model is an environment fact, not an infrastructure failure + i = src.find('result "model $name" SKIP 0') + if i != -1 and "infra=1" in src[i:i + 400]: + bad.append("the missing-model skip still raises infra (the gate would exit 2)") + return bad + + def script_invariants(): """The gate's lock placement, from the script itself (no GPU needed): engine#682 took ~/npu.lock off the GPU steps, because an NPU serve and the GPU run concurrently here (engine#673 measured ~87 % of the NPU @@ -111,6 +134,7 @@ def main(): threading.Thread(target=srv.serve_forever, daemon=True).start() url = f"http://127.0.0.1:{srv.server_address[1]}" bad = script_invariants() + bad += missing_model_is_a_skip() with tempfile.TemporaryDirectory() as d: gold = os.path.join(d, "g.json") STATE["mode"] = "good"