From d1a5f22c2fe415265a1288a4f25597e513fa325b Mon Sep 17 00:00:00 2001 From: hrx-compat lane Date: Sun, 15 Nov 2026 06:42:51 -0400 Subject: [PATCH] hw-gate: a model that is not on this runner SKIPs its row and is named at the end (engine#2501) A FAIL row for an absent GGUF reds every PR's gate for a reason no PR can fix -- the row was red on main's own pin (engine#2485 run 43677464548, engine#2487 run 43678134013), and the file is not on the box. The two options in the issue were "put the GGUF back" (a box change, needs the owner) and "SKIP, and say so"; this is the second, and engine#2494 argues against the first: that model is the crash-1 storm source for the MES resets. result "model $name" SKIP 0 "not on this runner: $model" missing_models="${missing_models:-} $name" ... note "::warning::model rows skipped because the GGUF is not on this runner:"$missing_models The skip deliberately does not set infra=1. A missing environment fact is not an infrastructure failure, and infra exits 2 -- which is the FAIL the row used to be, one indirection away. The second half is the one that matters more: coverage that did not happen has to be visible, or the matrix looks complete. One warning line at the end of the run names every model that was not exercised, so the difference between "the gate passed" and "the gate passed on what was here" is in the log. Test: tests/hw_gate_test.py gains missing_model_is_a_skip(), asserting from the script text (the same way the lock invariant is asserted -- no GPU, no models needed): * the FAIL form is gone; * the SKIP form is present with the path in the message; * the summary line names the skipped rows; * the skip does not raise infra within its branch. Both directions checked: against the old script it reports two named failures, against this one it passes. (The first wiring of the arm defined the function without calling it, and "passed" against both -- which is why the two-sided run was worth doing before shipping.) --- scripts/hw-gate.sh | 13 ++++++++++++- tests/hw_gate_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 51c46f447..310a5e036 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -635,7 +635,14 @@ if has kat || has needle || has kld; then while IFS='|' read -r name file repeat donee srvargs; do want_model "$name" || continue model=$MODELS/$file base=$STORE/kld/$name.kld - if [ ! -f "$model" ]; then result "model $name" FAIL 0 "model missing: $model"; infra=1; continue; fi + # engine#2501: a model file that is not on this runner is an ENVIRONMENT fact, not a gate result, and a FAIL + # row here reds every PR's gate for a reason no PR can fix. SKIP it and name the path -- and collect it, so the + # run ends with one line saying which models were not exercised instead of a silently smaller matrix. + if [ ! -f "$model" ]; then + result "model $name" SKIP 0 "not on this runner: $model" + missing_models="${missing_models:-} $name" + continue + fi dokat=0; has kat && dokat=1 needle=0; [ "$donee" = 1 ] && has needle && needle=1 dokld=0; has kld && [ -f "$TEXT" ] && [ -z "$srvargs" ] && dokld=1 @@ -802,6 +809,10 @@ rings=$(ring_timeouts) [ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$OUT/summary.md" >> "$GITHUB_STEP_SUMMARY" cat "$OUT/summary.md" if [ "$nfail" != 0 ]; then + # engine#2501: coverage that did not happen has to be visible on the run, or the matrix looks complete. + if [ -n "${missing_models:-}" ]; then + note "::warning::model rows skipped because the GGUF is not on this runner:"$missing_models + fi [ "$infra" = 1 ] && exit 2 exit 1 fi diff --git a/tests/hw_gate_test.py b/tests/hw_gate_test.py index 4609ad87b..a67f4d840 100644 --- a/tests/hw_gate_test.py +++ b/tests/hw_gate_test.py @@ -88,6 +88,29 @@ def kat(url, *args): return p.returncode, p.stdout +def missing_model_is_a_skip(): + """engine#2501: a GGUF that is not on this runner must SKIP its row and be named at the end, not FAIL it. + + The FAIL cost every PR's gate a row for a reason no PR could fix, and the row was red on main's own pin. The + opposite mistake is worse: a silently skipped model makes the matrix look complete, so the run has to say which + rows it did not exercise. Both halves are asserted from the script text, like the lock invariant above (no GPU, + no models: the behaviour under test is what the missing-file branch does).""" + bad = [] + with open(os.path.join(ROOT, "scripts", "hw-gate.sh")) as f: + src = f.read() + if 'result "model $name" FAIL 0 "model missing' in src: + bad.append("a missing GGUF still FAILs its row (engine#2501)") + if 'result "model $name" SKIP 0 "not on this runner' not in src: + bad.append("a missing GGUF does not SKIP its row with the path named") + if "missing_models=" not in src or "model rows skipped because the GGUF is not on this runner" not in src: + bad.append("skipped model rows are not named in the run summary") + # the skip must not set infra: a missing model is an environment fact, not an infrastructure failure + i = src.find('result "model $name" SKIP 0') + if i != -1 and "infra=1" in src[i:i + 400]: + bad.append("the missing-model skip still raises infra (the gate would exit 2)") + return bad + + def script_invariants(): """The gate's lock placement, from the script itself (no GPU needed): engine#682 took ~/npu.lock off the GPU steps, because an NPU serve and the GPU run concurrently here (engine#673 measured ~87 % of the NPU @@ -111,6 +134,7 @@ def main(): threading.Thread(target=srv.serve_forever, daemon=True).start() url = f"http://127.0.0.1:{srv.server_address[1]}" bad = script_invariants() + bad += missing_model_is_a_skip() with tempfile.TemporaryDirectory() as d: gold = os.path.join(d, "g.json") STATE["mode"] = "good"