From 2a2d10a30216708c537fb3e363d00094b745a67c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 10:18:36 -0400 Subject: [PATCH] gpu-hold: the take-time refusal says how old the newest fault is (engine#2542) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refusal said "N amdgpu/MES/amdxdna fault line(s) in the last 10 min", which cannot distinguish two different worlds: a device that is misbehaving now, and another lane's teardown that logged its faults nine minutes ago. On 2026-11-15 the second was the real case, three times over: burst 08:43:02 <- cl-npuperf-amdcol-h171-r1q15 released the NPU at 08:39:36 (+3m26s, token free) burst 08:53:38 <- cl-npuperf-amdcol-q25-decode released at 08:52:02 (+1m36s) op 18 09:11:10 <- cl-npuperf-amdcol-q25-hostgemm released at 09:02:41 (+8m29s) Every one landed while nothing held the device, and the guard would have refused any lane's ticket for ten minutes with a message that reads the same as a wedged GPU. The age comes from a second `journalctl -o short-unix` query taken only on the refusal path, so the printed lines keep their human-readable form: ::error::refusing: 1 amdgpu/MES/amdxdna fault line(s) in the last 10 min, newest 43204s ago - the ticket is released unheld. Inspect the kernel log, then retry This changes the MESSAGE, not the decision: a fault line still refuses the ticket. The semantics stay as #2480/#2498 left them, because "some faults arrive after a release" is a hypothesis from three correlations, not a licence to let a possibly-wedged device through. Arm added to tests/gpu_hold_fault_abort_test.sh: a stub journalctl supplies an epoch, and the arm asserts the refusal contains `newest s ago` and NOT a bare epoch (a raw 1700000000 in the message would be worse than nothing). Mutation: forcing `_age=""` (the pre-fix message) fails it by name — FAIL take-time age: the refusal does not say how old the newest line is Restored: `gpu_hold_fault_abort: ok (… default on -> 78 …)`. copyright rc 0, lane_syntax balanced. --- scripts/gpu-hold | 11 ++++++++++- tests/gpu_hold_fault_abort_test.sh | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 03c263a9c..2ea60db84 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -316,7 +316,16 @@ while [ "$state" = queued ]; do _resets=$(printf '%s\n' "$_klog" | grep -ciE "$_reset_re") if [ "${_resets:-0}" -gt 0 ]; then rm -f "$TOKEN"; state=done - log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min - the ticket is released unheld. Inspect the kernel log, then retry" + # HOW OLD is the newest one? "in the last 10 min" cannot tell a wedged device from another lane's + # teardown nine minutes ago, and on 2026-11-15 that distinction mattered: every amdxdna fault that + # refused a ticket arrived 1.5-8.5 min AFTER the previous ticket released the NPU, i.e. nothing was + # running when it was logged (engine#2542). The age is the datum a reader needs; the epoch comes + # from a second query so the printed lines keep their human-readable form. + _newest=$(journalctl -k --since "-${_reset_window} min" --no-pager -o short-unix -q 2> /dev/null | + grep -iE "$_reset_re" | tail -1 | awk '{print int($1)}') + _age="" + case "${_newest:-}" in ''|*[!0-9]*) ;; *) _age=", newest $(( $(date +%s) - _newest ))s ago" ;; esac + log "::error::refusing: ${_resets} amdgpu/MES/amdxdna fault line(s) in the last ${_reset_window} min${_age} - the ticket is released unheld. Inspect the kernel log, then retry" printf '%s\n' "$_klog" | grep -iE "$_reset_re" | tail -3 exit 78 fi diff --git a/tests/gpu_hold_fault_abort_test.sh b/tests/gpu_hold_fault_abort_test.sh index 523a85a9d..2ce68b6eb 100755 --- a/tests/gpu_hold_fault_abort_test.sh +++ b/tests/gpu_hold_fault_abort_test.sh @@ -115,6 +115,21 @@ take_run ring "kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* ring gfx_0.0.0 timeout take_run hang "kernel: amdgpu 0000:c5:00.0: amdgpu: GPU hang detected on ring gfx" [ "$trc" = 78 ] || { echo "FAIL take-time: an amdgpu hang -> rc $trc, want 78"; fail=1; } +# the refusal must say HOW OLD the newest line is: "in the last 10 min" cannot tell a wedged device from another lane's +# teardown nine minutes ago, and on 2026-11-15 every amdxdna fault that refused a ticket arrived 1.5-8.5 min AFTER the +# previous ticket released the NPU, with nothing running (engine#2542). A stub journalctl supplies the epoch. +mkdir -p "$scratch/agebin" +printf '#!/bin/bash\nprintf "1700000000 kernel: amdgpu 0000:c5:00.0: [drm] *ERROR* amdgpu: GPU reset begin!\\n"\n' > "$scratch/agebin/journalctl" +chmod +x "$scratch/agebin/journalctl" +q=$scratch/tqage; mkdir -p "$q"; rm -f "$scratch/age-token" +env PATH="$scratch/agebin:$PATH" HOME=$scratch ONEBIT_HOLD_DIR="$scratch/agehold" GPU_HOLD_COOLDOWN=0 \ + GPU_HOLD_WAIT=5 GPU_HOLD_MINUTES=1 GPU_HOLD_THERMAL_FLAG="$scratch/x" GPU_HOLD_RESET_WINDOW_MIN=10 \ + timeout 60 "$hold" -l takeage -- true > "$scratch/tlogage" 2>&1 +age_rc=$? +[ "$age_rc" = 78 ] || { echo "FAIL take-time age: rc $age_rc, want 78"; fail=1; } +grep -qE 'newest [0-9]+s ago' "$scratch/tlogage" || { echo "FAIL take-time age: the refusal does not say how old the newest line is"; fail=1; } +grep -qE 'newest [0-9]{9,}s ago' "$scratch/tlogage" && { echo "FAIL take-time age: the age printed in seconds, not an epoch"; fail=1; } + take_run clean clean [ "$trc" = 0 ] || { echo "FAIL take-time: a clean log -> rc $trc, want 0 (a healthy hold must still run)"; fail=1; }