diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b917f82e..45b5a2f93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -629,6 +629,7 @@ add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_tes # engine#2500/#2513: a failing row's kept server log gets a name unique per failure (CI's $OUT basename is constant, so # the first version overwrote the previous run's evidence), and a failed copy names nothing. No GPU, no ticket. add_test(NAME hw_gate_kept_log COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_kept_log_test.sh ${CMAKE_SOURCE_DIR}) +add_test(NAME hw_gate_phase_rows COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_phase_rows_test.sh ${CMAKE_SOURCE_DIR}) # engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. # The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any # exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer. diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 8a75101ce..e77416760 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -589,7 +589,15 @@ if has ops; then # pass". That is a *filter* problem, not a kernel result - and it must never read as a pass either. # Report it as SKIP with the selection that produced it, so neither an empty run nor a broken filter # can be mistaken for a measured one. + # infra=1: nothing was tested, so this is "could not run", not a measured SKIP. Without the flag a filter + # that matches no case leaves the gate reporting PASS (engine#2522 set it on the absent-input SKIPs for the + # same reason; the zero-cases SKIP was the one that was missed). + infra=1 result ops SKIP $(( $(date +%s) - t0 )) "no case matched -b HRX0 -o \"$OPS\": the zero-cases guard refused to report a pass (fork engine#657) - fix the filter or the build" + # engine#2522's rule, applied to this row: a SKIP means the run did not test what it claims to test, so the run + # as a whole is infra-incomplete and must not read as PASS. The verdict block now consults infra, so setting it + # here is what turns "the ops phase had nothing to run" into a non-green gate instead of a quiet row. + infra=1 else unknown=$(printf '%s\n' "$fails" | grep -v -F "$FLAKY_CASES" | grep -c . || true) st=FAIL detail="${passed:-rc=$rc}: $(printf '%s' "$fails" | head -n 3 | tr '\n' ' ')" @@ -643,8 +651,11 @@ if has ops; then fi fi -# ---- kat + needle + KLD: one step per model ----------------------------------------------- -if has kat || has needle || has kld; then +# ---- kat + needle + KLD + placement: one step per model ------------------------------------ +# `placement` must be in this guard. It was not, so `--phases build,placement` (and `placement` alone) entered nothing, +# produced no row, and the gate reported PASS over a run that never asked the question (2026-11-15, dispatch 43700775742: +# "PHASES: build,placement" then a summary table with `build` and nothing else, `## HRX hardware gate: PASS`). +if has kat || has needle || has kld || has placement; then [ -f "$TEXT" ] || { has kld && { result kld SKIP 0 "text missing (input absent): $TEXT"; infra=1; }; } while IFS='|' read -r name file repeat donee srvargs; do want_model "$name" || continue @@ -796,10 +807,38 @@ fi # ---- summary --------------------------------------------------------------------------------- total=$(( $(date +%s) - START )) +# A REQUESTED PHASE THAT PRODUCED NO ROW IS A FAILURE, not a pass. `--phases placement` entered no model loop until the +# guard above was fixed, and the gate reported `## HRX hardware gate: PASS` over a run that never asked the question +# (dispatch 43700775742). Whatever the next such slip is, it must not be able to read as green: every phase named on the +# command line has to appear as a row (PASS, FAIL, SKIP, or a `model …` row), or the gate says so by name. +for _ph in $(printf '%s' "$PHASES" | tr ',' ' '); do + case "$_ph" in ''|*' '*) continue ;; esac + # The per-model phases report their missing/throttled models as a `model ` row, so that row satisfies the + # request for any of them; only a phase that produced NO row at all is a slip. + # `placement` has its OWN row kind, so a `model …` row must not satisfy a request for it: with the prefix rule + # below covering all four phases, `--phases placement` was satisfied by any model row and a run with no placement + # row at all still read as green -- the same defect one level in (review at 9dd6077). The per-model phases DO + # report their missing/throttled models as `model ` rows, so only those three accept that alternative. + # A phase is the ROW'S FIRST FIELD, not a prefix of it: `npu` is a prefix of `npuserve`, so `^npu` read an + # `npuserve` row as proof that `npu` had run (review of engine#2523, measured on a results file holding only + # `build` and `npuserve`). Rows are `namestatus...`, so `[[:space:]]` is what makes it a field match. + case "$_ph" in + kat|kld|needle) _phre="^(${_ph}|model)[[:space:]]" ;; + *) _phre="^${_ph}[[:space:]]" ;; + esac + grep -qE "$_phre" "$RESULTS" 2>/dev/null || + printf '%s\tFAIL\t0\trequested but produced no result row: --phases %s ran nothing for it\n' "$_ph" "$_ph" >> "$RESULTS" +done nfail=$(awk -F'\t' '$2 == "FAIL"' "$RESULTS" | wc -l) rings=$(ring_timeouts) { - echo "## HRX hardware gate: $([ "$nfail" = 0 ] && echo PASS || echo "FAIL ($nfail)")" + # A run that could not test an input is NOT a pass. `infra=1` marks that (an absent model, absent text, a refused + # memory pre-check); before this, the flag was only consulted inside `if [ "$nfail" != 0 ]`, so once an absent input + # became a SKIP row (engine#2522) a run with nothing to test reported `## HRX hardware gate: PASS` and exited 0. + _verdict=PASS + [ "$nfail" != 0 ] && _verdict="FAIL ($nfail)" + [ "$infra" = 1 ] && [ "$_verdict" = PASS ] && _verdict="INCOMPLETE (an input was absent: nothing was tested)" + echo "## HRX hardware gate: $_verdict" echo echo "Commit \`$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null)\`, llama.cpp \`$(git -C "$ROOT" ls-tree HEAD third_party/llama.cpp | awk '{print substr($3,1,12)}')\`, hrx-system \`$(git -C "$ROOT" ls-tree HEAD third_party/hrx-system | awk '{print substr($3,1,12)}')\`, binaries \`$BIN\`, ${total}s, Tctl at end $(tctl) C, amdgpu ring timeouts during the run: $rings." echo @@ -813,8 +852,6 @@ rings=$(ring_timeouts) } > "$OUT/summary.md" [ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$OUT/summary.md" >> "$GITHUB_STEP_SUMMARY" cat "$OUT/summary.md" -if [ "$nfail" != 0 ]; then - [ "$infra" = 1 ] && exit 2 - exit 1 -fi +[ "$infra" = 1 ] && exit 2 # not green: an input was absent, so the gate did not test what it was asked to +[ "$nfail" != 0 ] && exit 1 exit 0 diff --git a/tests/hw_gate_phase_rows_test.sh b/tests/hw_gate_phase_rows_test.sh new file mode 100755 index 000000000..a0bfe3232 --- /dev/null +++ b/tests/hw_gate_phase_rows_test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# hw-gate keeps a failing row's server log (engine#2500, #2513) under $HW_GATE_STORE/failed and names it in the +# failure line. The name must be unique per failure: CI sets HW_GATE_OUT=$RUNNER_TEMP/hw-gate, so a name built from +# basename "$OUT" was the constant "hw-gate-.log" and every failing run on every PR overwrote the last one's +# evidence. These arms run the CI shape (same model, same run, same second), and check that a failed copy prints no +# path. A recording fake gpu-hold is first on PATH, so the internal mode is also proven to take no GPU ticket. +# No GPU, no model, runs in CI. +# +# The gate's phase check must match the phase FIELD, not a prefix of it, and only the phases that actually write +# `model ` rows may be satisfied by one (review of engine#2523): +# +# * `npu` is a prefix of `npuserve`, so `grep -qE "^npu"` read an `npuserve` row as proof that `npu` had run; +# * `placement` writes `placement `, not `model `, so grouping it with kat|kld|needle let any model row +# satisfy a requested placement phase -- the same silent pass, one name over. +# +# The loop is EXTRACTED FROM THE SCRIPT and run, so this pins the shipped code rather than a copy of it. A copy would +# pass while the real check drifted. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +gate=$root/scripts/hw-gate.sh +fail=0 + +# the check, verbatim, as it ships +snippet=$(mktemp) +sed -n '/^for _ph in \$(printf .%s. "\$PHASES"/,/^done$/p' "$gate" > "$snippet" +[ -s "$snippet" ] || { echo "FAIL: the phase-check loop is no longer where this test finds it"; exit 1; } + +check() { # $1 results file, $2 PHASES -> prints the FAIL rows the check appended + ( PHASES=$2; RESULTS=$1; . "$snippet" ) 2>&1 + awk -F'\t' '$2 == "FAIL" {print $1}' "$1" +} + +run() { # name, results content, phases, expected-substring-in-output + local t; t=$(mktemp) + printf '%b' "$2" > "$t" + local out; out=$(check "$t" "$3") + rm -f "$t" + if [ -n "$4" ]; then + case "$out" in *"$4"*) ;; *) echo "FAIL $1: expected a FAIL row for '$4', got '${out:-}'"; fail=1; return ;; esac + else + [ -z "$out" ] || { echo "FAIL $1: expected no failure, got '$out'"; fail=1; return; } + fi + echo "ok $1" +} + +# the two slips the review measured, both must now fail +run "a npuserve row does not satisfy a requested npu phase" \ + 'build\tPASS\t1\tok\nnpuserve\tPASS\t1\tok\n' "build,npu,npuserve" "npu" +run "a model row does not satisfy a requested placement phase" \ + 'build\tPASS\t1\tok\nmodel gemma\tPASS\t1\tok\n' "build,placement" "placement" + +# and the three legitimate satisfactions must keep working +run "a placement row satisfies placement" 'build\tPASS\t1\tok\nplacement gemma\tPASS\t1\tok\n' "build,placement" "" +run "a model row still satisfies kat" 'build\tPASS\t1\tok\nmodel gemma\tPASS\t1\tok\n' "build,kat" "" +run "a plain row satisfies its own phase" 'build\tPASS\t1\tok\nops\tPASS\t1\tok\n' "build,ops" "" + +rm -f "$snippet" +[ "$fail" = 0 ] && echo "hw_gate_phase_rows: ok (a phase is a field, not a prefix; only kat/kld/needle accept a model row)" +exit "$fail"