From 89484ff693310713183aa99075474177413c66cb Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 19:02:06 -0400 Subject: [PATCH] Put the accel sampler in the repo, with its self-test as a ctest row It was living only in ~/.cache/accel-sampler, which means a reboot loses it and the 24-hour window starts over -- and more to the point, nobody else can run it. The point of engine#2542 is to attribute a burst, and that needs an instrument someone other than its author can invoke. So: tools/diag/accel_sampler.sh, same code, with the self-test documented in its header and registered as ctest `accel_sampler`. Four arms plus rotation, all green, and the rotation arm was proved able to fail by disabling rotation. The header says why the self-test matters in the words of this issue: a silent correlator loses exactly the event it exists to catch, and that mistake was made twice in one day here. Read-only by construction -- /proc fd symlinks and journald, no sudo, no device access, no XRT, never :18097. --- CMakeLists.txt | 3 + tools/diag/accel_sampler.sh | 183 ++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 tools/diag/accel_sampler.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a99e7833..5074f74e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -635,6 +635,9 @@ add_test(NAME hw_gate_mem COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_mem_tes # the first version overwrote the previous run's evidence), and a failed copy names nothing. No GPU, no ticket. add_test(NAME hw_gate_kept_log COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_kept_log_test.sh ${CMAKE_SOURCE_DIR}) add_test(NAME hw_gate_phase_rows COMMAND bash ${CMAKE_SOURCE_DIR}/tests/hw_gate_phase_rows_test.sh ${CMAKE_SOURCE_DIR}) +# engine#2542: the accel sampler's self-test -- the correlation and rotation arms must be green before anyone +# starts a long capture, because a silent correlator loses exactly the event it exists to catch. +add_test(NAME accel_sampler COMMAND bash ${CMAKE_SOURCE_DIR}/tools/diag/accel_sampler.sh selftest) # engine#2523's rule reached main without its runtime arm: the source-reading hw_gate_test.py asserts the SKIP rows' # text and their `infra=1`, and hw_gate_phase_rows asserts phase parsing, but nothing ran the gate end to end with a # backend that reports "no test case ran". That is the case the fix was for -- zero cases must read INCOMPLETE and exit diff --git a/tools/diag/accel_sampler.sh b/tools/diag/accel_sampler.sh new file mode 100755 index 000000000..ab569071a --- /dev/null +++ b/tools/diag/accel_sampler.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# hw-gate keeps a failing row's server log (engine#2500, #2513) under $HW_GATE_STORE/failed and names it in the +# failure line. The name must be unique per failure: CI sets HW_GATE_OUT=$RUNNER_TEMP/hw-gate, so a name built from +# basename "$OUT" was the constant "hw-gate-.log" and every failing run on every PR overwrote the last one's +# evidence. These arms run the CI shape (same model, same run, same second), and check that a failed copy prints no +# path. A recording fake gpu-hold is first on PATH, so the internal mode is also proven to take no GPU ticket. +# No GPU, no model, runs in CI. +# +# The gate's phase check must match the phase FIELD, not a prefix of it, and only the phases that actually write +# `model ` rows may be satisfied by one (review of engine#2523): +# +# * `npu` is a prefix of `npuserve`, so `grep -qE "^npu"` read an `npuserve` row as proof that `npu` had run; +# * `placement` writes `placement `, not `model `, so grouping it with kat|kld|needle let any model row +# satisfy a requested placement phase -- the same silent pass, one name over. +# +# The loop is EXTRACTED FROM THE SCRIPT and run, so this pins the shipped code rather than a copy of it. A copy would +# pass while the real check drifted. +# engine#2542: attribute the amdxdna IO_PAGE_FAULT bursts by sampling WHICH PROCESS HOLDS /dev/accel/accel0 open, and +# correlating each fault with the samples around it. +# +# Read-only by construction: it scans /proc for open-file symlinks and reads journald. It does not open the accel node, +# does not talk to XRT, never touches :18097, and needs no sudo (so no `lsof` loops). Because it takes no device action it +# needs no npu-hold ticket -- but it also must never *cause* a fault, which is why every read is /proc or the journal. +# +# Two loops in one unit: +# sample every SAMPLE_S seconds: the PIDs holding the accel node, with cmdline, cgroup scope and the npu-hold ticket +# (from ~/NPU-TOKEN) as they stood at that moment +# watch journalctl -k -f, and for each IO_PAGE_FAULT line, the samples within +-WINDOW_S seconds, written as one +# attribution record +# selftest proves the correlation and the rotation WITHOUT needing a fault -- run it before a long capture, because a +# silent correlator loses the event it exists to catch (that mistake was made twice in one day on this issue) +# +# Inputs (env, all optional): ACCEL_SAMPLER_DIR, SAMPLE_S (2), WINDOW_S (5), MAX_BYTES (50 MB), ACCEL_NODE. +set -u +DIR=${ACCEL_SAMPLER_DIR:-$HOME/.cache/accel-sampler} # override to run a second instance +SAMPLE_S=${SAMPLE_S:-2} +FULL_SCAN_S=${FULL_SCAN_S:-30} # a full /proc sweep every 2 s cost 62% of a core (measured); holders are long-lived + # servers, so re-checking the known ones each sample is enough between sweeps +WINDOW_S=${WINDOW_S:-5} +MAX_BYTES=${MAX_BYTES:-52428800} # 50 MB, rotate rather than grow without bound +NODE=${ACCEL_NODE:-/dev/accel/accel0} +mkdir -p "$DIR" || exit 1 +SAMP=$DIR/samples.log +ATTR=$DIR/attributed.log +mkdir -p "$DIR" + +rotate() { # $1 = file: hard cap, one previous kept + local f=$1 sz + sz=$(stat -c %s "$f" 2>/dev/null || echo 0) + if [ "$sz" -gt "$MAX_BYTES" ]; then + mv -f "$f" "$f.1" 2>/dev/null + : > "$f" + fi +} + +ticket() { # the npu-hold ticket that was live, if one is recorded + for p in "$HOME/NPU-TOKEN" "$HOME/.npu-hold" "$HOME/lb/NPU-TOKEN"; do + [ -f "$p" ] && { head -c 200 "$p" | tr '\n' ' '; return; } + done + echo "-" +} + +holders() { # print "pid|cmd|scope|fds" for every process holding $NODE open + local pid fd tgt cmd scope n + for pid in /proc/[0-9]*; do + pid=${pid#/proc/} + [ -d "/proc/$pid/fd" ] || continue # kernel threads: no fd table, skip cheaply + n=0 + for fd in /proc/"$pid"/fd/*; do + tgt=$(readlink "$fd" 2>/dev/null) || continue + [ "$tgt" = "$NODE" ] && n=$((n + 1)) + done + [ "$n" -eq 0 ] && continue + cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-120) + scope=$(sed -n 's/^0:://p' "/proc/$pid/cgroup" 2>/dev/null | head -1) + [ -z "$scope" ] && scope=$(awk -F/ '/^0::/{print $NF}' "/proc/$pid/cgroup" 2>/dev/null) + printf '%s|%s|%s|%s\n' "$pid" "${cmd:--}" "${scope:--}" "$n" + done +} + +KNOWN=$DIR/known.pids +sample_once() { # one line per (moment, holder); "none" when nothing holds the node + local ts seen=0 now last=0 + ts=$(date +%s) + now=$ts + [ -f "$KNOWN" ] && last=$(cat "$KNOWN.last" 2>/dev/null || echo 0) + if [ $((now - last)) -ge "$FULL_SCAN_S" ]; then + holders | cut -d'|' -f1 | sort -u > "$KNOWN.new" + mv -f "$KNOWN.new" "$KNOWN"; echo "$now" > "$KNOWN.last" + fi + # only the processes known to hold the node: count their accel fds (cheap) and record them + while read -r pid; do + [ -z "$pid" ] && continue + local n=0 fd + for fd in /proc/"$pid"/fd/*; do + [ "$(readlink "$fd" 2>/dev/null)" = "$NODE" ] && n=$((n + 1)) + done + [ "$n" -eq 0 ] && continue # it closed it; the next full sweep drops it from KNOWN + seen=1 + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$ts" "$pid" "$n" \ + "$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-120)" \ + "$(awk -F/ '/^0::/{print $NF}' "/proc/$pid/cgroup" 2>/dev/null)" \ + "$(ticket)" >> "$SAMP" + done < "$KNOWN" 2>/dev/null + if [ "$seen" = 0 ]; then + printf '%s\t-\t-\t-\t-\t%s\n' "$ts" "$(ticket)" >> "$SAMP" + fi + rotate "$SAMP" +} + +correlate() { # $1 lo $2 hi -> "pid:cmd:scope" for every sample in [lo,hi], deduplicated + awk -F'\t' -v lo="$1" -v hi="$2" '$1 >= lo && $1 <= hi' "$SAMP" 2>/dev/null | cut -f2,4,5 | sort -u | tr '\n' ';' +} + +selftest() { # prove the correlation before an event needs it: the window, the fields, and the miss case + local d fail=0 out + d=$(mktemp -d "$DIR/selftest.XXXXXX") # never /tmp: this lane's rule, and tmpfs pressure has bitten the box + printf '1000\t111\t1\t/serve A\t/scope/a\tticket1\n1010\t222\t2\t/serve B\t/scope/b\tticket2\n' > "$d/samples.log" + SAMP=$d/samples.log + out=$(correlate 1005 1015) # +-5 s around a fault at 1010 + case "$out" in *222*) ;; *) echo "FAIL: the holder at the fault instant is missing"; fail=1 ;; esac + case "$out" in *111*) echo "FAIL: a sample outside the window was included"; fail=1 ;; esac + out=$(correlate 2000 2010) + [ -z "$out" ] || { echo "FAIL: a window with no samples produced '$out'"; fail=1; } + # a "none" sample (nobody holding the node) must be reported as such, not as a holder + printf '1000\t-\t-\t-\t-\t-\n' > "$d/samples.log" + out=$(correlate 995 1005) + case "$out" in *'-:-:-'*|*'-'*) ;; *) echo "FAIL: a no-holder sample was not passed through"; fail=1 ;; esac + # rotation: a 24 h run WILL hit it, so it gets an arm rather than a hand test + MAX_BYTES=10; printf 'line one is long enough to exceed the cap\n' > "$d/rot.log" + rotate "$d/rot.log" + [ -f "$d/rot.log.1" ] || { echo "FAIL: rotation did not keep a previous file"; fail=1; } + [ "$(stat -c %s "$d/rot.log")" -eq 0 ] || { echo "FAIL: the live file was not truncated"; fail=1; } + rm -rf "$d" + [ "$fail" = 0 ] && echo "accel-sampler self-test: ok (window, fields, empty window, no-holder sample, rotation)" + return "$fail" +} + +watch_faults() { # correlate each fault with the samples around it + local line ts lo hi hits + journalctl -k -f -o short-unix 2>/dev/null | grep --line-buffered 'IO_PAGE_FAULT' | + while IFS= read -r line; do + ts=${line%% *} # short-unix: " host kernel: ..." + ts=${ts%%.*} + lo=$((ts - WINDOW_S)); hi=$((ts + WINDOW_S)) + hits=$(correlate "$lo" "$hi") + printf '%s\t%s\t%s\n' "$ts" "${hits:--}" "$(printf '%s' "$line" | cut -c1-200)" >> "$ATTR" + rotate "$ATTR" + done +} + +case "${1:-both}" in + sample) sample_once ;; + watch) watch_faults ;; + both) + # the fault reader first, so no fault is missed between start and the first sample + watch_faults & + W=$! + trap 'kill "$W" 2>/dev/null' EXIT INT TERM + while :; do + sample_once + sleep "$SAMPLE_S" + done + ;; + holders) holders ;; + selftest|--self-test) selftest ;; + *) echo "usage: $0 [both|sample|watch|holders|selftest]" >&2; exit 2 ;; +esac