Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,7 @@ add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_tes
# engine#2500/#2513: a failing row's kept server log gets a name unique per failure (CI's $OUT basename is constant, so
# the first version overwrote the previous run's evidence), and a failed copy names nothing. No GPU, no ticket.
add_test(NAME hw_gate_kept_log COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_kept_log_test.sh ${CMAKE_SOURCE_DIR})
add_test(NAME hw_gate_phase_rows COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_phase_rows_test.sh ${CMAKE_SOURCE_DIR})
# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy.
# The gate's "is the coder busy?" probe: `0` idle, `N` busy, `-1` cannot tell. The parser answered `0` on any
# exception while `curl -s` exits 0 on a 404, so a wrong URL, a refused connection and an idle coder were one answer.
Expand Down
51 changes: 44 additions & 7 deletions scripts/hw-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,15 @@ if has ops; then
# pass". That is a *filter* problem, not a kernel result - and it must never read as a pass either.
# Report it as SKIP with the selection that produced it, so neither an empty run nor a broken filter
# can be mistaken for a measured one.
# infra=1: nothing was tested, so this is "could not run", not a measured SKIP. Without the flag a filter
# that matches no case leaves the gate reporting PASS (engine#2522 set it on the absent-input SKIPs for the
# same reason; the zero-cases SKIP was the one that was missed).
infra=1
result ops SKIP $(( $(date +%s) - t0 )) "no case matched -b HRX0 -o \"$OPS\": the zero-cases guard refused to report a pass (fork engine#657) - fix the filter or the build"
# engine#2522's rule, applied to this row: a SKIP means the run did not test what it claims to test, so the run
# as a whole is infra-incomplete and must not read as PASS. The verdict block now consults infra, so setting it
# here is what turns "the ops phase had nothing to run" into a non-green gate instead of a quiet row.
infra=1
else
unknown=$(printf '%s\n' "$fails" | grep -v -F "$FLAKY_CASES" | grep -c . || true)
st=FAIL detail="${passed:-rc=$rc}: $(printf '%s' "$fails" | head -n 3 | tr '\n' ' ')"
Expand Down Expand Up @@ -643,8 +651,11 @@ if has ops; then
fi
fi

# ---- kat + needle + KLD: one step per model -----------------------------------------------
if has kat || has needle || has kld; then
# ---- kat + needle + KLD + placement: one step per model ------------------------------------
# `placement` must be in this guard. It was not, so `--phases build,placement` (and `placement` alone) entered nothing,
# produced no row, and the gate reported PASS over a run that never asked the question (2026-11-15, dispatch 43700775742:
# "PHASES: build,placement" then a summary table with `build` and nothing else, `## HRX hardware gate: PASS`).
if has kat || has needle || has kld || has placement; then
[ -f "$TEXT" ] || { has kld && { result kld SKIP 0 "text missing (input absent): $TEXT"; infra=1; }; }
while IFS='|' read -r name file repeat donee srvargs; do
want_model "$name" || continue
Expand Down Expand Up @@ -796,10 +807,38 @@ fi

# ---- summary ---------------------------------------------------------------------------------
total=$(( $(date +%s) - START ))
# A REQUESTED PHASE THAT PRODUCED NO ROW IS A FAILURE, not a pass. `--phases placement` entered no model loop until the
# guard above was fixed, and the gate reported `## HRX hardware gate: PASS` over a run that never asked the question
# (dispatch 43700775742). Whatever the next such slip is, it must not be able to read as green: every phase named on the
# command line has to appear as a row (PASS, FAIL, SKIP, or a `model …` row), or the gate says so by name.
for _ph in $(printf '%s' "$PHASES" | tr ',' ' '); do
case "$_ph" in ''|*' '*) continue ;; esac
# The per-model phases report their missing/throttled models as a `model <name>` row, so that row satisfies the
# request for any of them; only a phase that produced NO row at all is a slip.
# `placement` has its OWN row kind, so a `model …` row must not satisfy a request for it: with the prefix rule
# below covering all four phases, `--phases placement` was satisfied by any model row and a run with no placement
# row at all still read as green -- the same defect one level in (review at 9dd6077). The per-model phases DO
# report their missing/throttled models as `model <name>` rows, so only those three accept that alternative.
# A phase is the ROW'S FIRST FIELD, not a prefix of it: `npu` is a prefix of `npuserve`, so `^npu` read an
# `npuserve` row as proof that `npu` had run (review of engine#2523, measured on a results file holding only
# `build` and `npuserve`). Rows are `name<TAB>status...`, so `[[:space:]]` is what makes it a field match.
case "$_ph" in
kat|kld|needle) _phre="^(${_ph}|model)[[:space:]]" ;;
*) _phre="^${_ph}[[:space:]]" ;;
esac
grep -qE "$_phre" "$RESULTS" 2>/dev/null ||
printf '%s\tFAIL\t0\trequested but produced no result row: --phases %s ran nothing for it\n' "$_ph" "$_ph" >> "$RESULTS"
done
nfail=$(awk -F'\t' '$2 == "FAIL"' "$RESULTS" | wc -l)
rings=$(ring_timeouts)
{
echo "## HRX hardware gate: $([ "$nfail" = 0 ] && echo PASS || echo "FAIL ($nfail)")"
# A run that could not test an input is NOT a pass. `infra=1` marks that (an absent model, absent text, a refused
# memory pre-check); before this, the flag was only consulted inside `if [ "$nfail" != 0 ]`, so once an absent input
# became a SKIP row (engine#2522) a run with nothing to test reported `## HRX hardware gate: PASS` and exited 0.
_verdict=PASS
[ "$nfail" != 0 ] && _verdict="FAIL ($nfail)"
[ "$infra" = 1 ] && [ "$_verdict" = PASS ] && _verdict="INCOMPLETE (an input was absent: nothing was tested)"
echo "## HRX hardware gate: $_verdict"
echo
echo "Commit \`$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null)\`, llama.cpp \`$(git -C "$ROOT" ls-tree HEAD third_party/llama.cpp | awk '{print substr($3,1,12)}')\`, hrx-system \`$(git -C "$ROOT" ls-tree HEAD third_party/hrx-system | awk '{print substr($3,1,12)}')\`, binaries \`$BIN\`, ${total}s, Tctl at end $(tctl) C, amdgpu ring timeouts during the run: $rings."
echo
Expand All @@ -813,8 +852,6 @@ rings=$(ring_timeouts)
} > "$OUT/summary.md"
[ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$OUT/summary.md" >> "$GITHUB_STEP_SUMMARY"
cat "$OUT/summary.md"
if [ "$nfail" != 0 ]; then
[ "$infra" = 1 ] && exit 2
exit 1
fi
[ "$infra" = 1 ] && exit 2 # not green: an input was absent, so the gate did not test what it was asked to
[ "$nfail" != 0 ] && exit 1
exit 0
74 changes: 74 additions & 0 deletions tests/hw_gate_phase_rows_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# hw-gate keeps a failing row's server log (engine#2500, #2513) under $HW_GATE_STORE/failed and names it in the
# failure line. The name must be unique per failure: CI sets HW_GATE_OUT=$RUNNER_TEMP/hw-gate, so a name built from
# basename "$OUT" was the constant "hw-gate-<model>.log" and every failing run on every PR overwrote the last one's
# evidence. These arms run the CI shape (same model, same run, same second), and check that a failed copy prints no
# path. A recording fake gpu-hold is first on PATH, so the internal mode is also proven to take no GPU ticket.
# No GPU, no model, runs in CI.
#
# The gate's phase check must match the phase FIELD, not a prefix of it, and only the phases that actually write
# `model <name>` rows may be satisfied by one (review of engine#2523):
#
# * `npu` is a prefix of `npuserve`, so `grep -qE "^npu"` read an `npuserve` row as proof that `npu` had run;
# * `placement` writes `placement <name>`, not `model <name>`, so grouping it with kat|kld|needle let any model row
# satisfy a requested placement phase -- the same silent pass, one name over.
#
# The loop is EXTRACTED FROM THE SCRIPT and run, so this pins the shipped code rather than a copy of it. A copy would
# pass while the real check drifted.
set -u
root=$(cd "$(dirname "$0")/.." && pwd)
gate=$root/scripts/hw-gate.sh
fail=0

# the check, verbatim, as it ships
snippet=$(mktemp)
sed -n '/^for _ph in \$(printf .%s. "\$PHASES"/,/^done$/p' "$gate" > "$snippet"
[ -s "$snippet" ] || { echo "FAIL: the phase-check loop is no longer where this test finds it"; exit 1; }

check() { # $1 results file, $2 PHASES -> prints the FAIL rows the check appended
( PHASES=$2; RESULTS=$1; . "$snippet" ) 2>&1
awk -F'\t' '$2 == "FAIL" {print $1}' "$1"
}

run() { # name, results content, phases, expected-substring-in-output
local t; t=$(mktemp)
printf '%b' "$2" > "$t"
local out; out=$(check "$t" "$3")
rm -f "$t"
if [ -n "$4" ]; then
case "$out" in *"$4"*) ;; *) echo "FAIL $1: expected a FAIL row for '$4', got '${out:-<none>}'"; fail=1; return ;; esac
else
[ -z "$out" ] || { echo "FAIL $1: expected no failure, got '$out'"; fail=1; return; }
fi
echo "ok $1"
}

# the two slips the review measured, both must now fail
run "a npuserve row does not satisfy a requested npu phase" \
'build\tPASS\t1\tok\nnpuserve\tPASS\t1\tok\n' "build,npu,npuserve" "npu"
run "a model row does not satisfy a requested placement phase" \
'build\tPASS\t1\tok\nmodel gemma\tPASS\t1\tok\n' "build,placement" "placement"

# and the three legitimate satisfactions must keep working
run "a placement row satisfies placement" 'build\tPASS\t1\tok\nplacement gemma\tPASS\t1\tok\n' "build,placement" ""
run "a model row still satisfies kat" 'build\tPASS\t1\tok\nmodel gemma\tPASS\t1\tok\n' "build,kat" ""
run "a plain row satisfies its own phase" 'build\tPASS\t1\tok\nops\tPASS\t1\tok\n' "build,ops" ""

rm -f "$snippet"
[ "$fail" = 0 ] && echo "hw_gate_phase_rows: ok (a phase is a field, not a prefix; only kat/kld/needle accept a model row)"
exit "$fail"
Loading