From 65397314787ee5b8da604259b633e2af803dfa2d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 06:21:40 -0400 Subject: [PATCH] hw-gate: record the ACTUAL GTT and MemAvailable at failure time (engine#2500 ask 2) The memory pre-check records the PREDICTED need -- the number that was satisfiable. What a failing row needs is what the box looked like when it died: a load that runs out in the allocator shows GTT near the ceiling, and that is attributable in one line instead of three hypotheses. On FAIL the result line now carries: [GTT now 39.2 GiB, MemAvailable 58.0 GiB] [log: ~/.cache/hw-gate/failed/-.log] Both reads verified on this box: /sys/class/drm/card*/device/mem_info_gtt_used (39.2 GiB at the time of writing) and /proc/meminfo MemAvailable (58.0 GiB). Absent values print '?' rather than 0.0, so a missing sysfs entry cannot read as an idle box. --- scripts/hw-gate.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 9b890c688..104f20b60 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -672,7 +672,13 @@ if has kat || has needle || has kld; then # failure line so a reader does not have to guess. The name is unique per failure (keep_failed_log). kept=$(keep_failed_log "$OUT/server-$name.log" "$name") [ -n "$kept" ] && echo "hw-gate: kept the failing server log at $kept" >&2 - result "model $name" FAIL "$secs" "server failed (rc $rc): $(tail -n 3 "$OUT/server-$name.log" 2>/dev/null | tr '\n' ' ' | cut -c1-200) [log: ${kept:-$OUT/server-$name.log}]" + # engine#2500 ask 2: at failure time, record what the box ACTUALLY looked like -- the pre-check + # records the predicted need, which is the number that was satisfiable. A load that dies in the + # allocator shows up here as GTT near the ceiling, and that is attributable in one line. + gtt=$(awk '{printf "%.1f", $1/1073741824}' /sys/class/drm/card*/device/mem_info_gtt_used 2>/dev/null | head -1) + avail=$(awk '/MemAvailable/{printf "%.1f", $2/1048576}' /proc/meminfo 2>/dev/null) + box="GTT now ${gtt:-?} GiB, MemAvailable ${avail:-?} GiB" + result "model $name" FAIL "$secs" "server failed (rc $rc): $(tail -n 3 "$OUT/server-$name.log" 2>/dev/null | tr '\n' ' ' | cut -c1-200) [$box] [log: ${kept:-$OUT/server-$name.log}]" continue fi if [ $dokat = 1 ]; then