Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,8 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test.
# wedged in uninterruptible sleep, which user space cannot create, so this covers the other half: a holder that released with
# a survivor must leave a record, and the NEXT holder must warn about it and clear it.
add_test(NAME gpu_hold_survivors COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_survivors_test.sh)
add_test(NAME gpu_hold_group
COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_group_test.sh)
# engine#1594: every target ci.yml's `cmake --build --target …` list names must exist in the configuration ci.yml
# creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja
# rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the
Expand Down
26 changes: 25 additions & 1 deletion scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -313,16 +313,40 @@ kill_tree() {
# ticket. The KILL pass then cannot see it and the orphan survives the hold -- which is the failure this function was
# written to end. A snapshot also fixes the grace period: `kill -0` with a pid list is true while ANY of them lives,
# so a wrapper that exits promptly no longer denies its children the grace they were promised.
# The group first: it is named by the payload's own pid and stays valid while ANY member lives, so this reaches
# descendants the walk cannot see (the root already gone, or a child that reparented before the snapshot). The
# walk stays as the fallback for a payload that moved itself into another group.
kill -TERM -- "-$root" 2> /dev/null
local tree; tree=$(descendants "$root")
for p in $tree; do kill -TERM "$p" 2> /dev/null; done
while kill -0 $tree 2> /dev/null && [ "$i" -lt "$grace" ]; do
sleep 1
i=$((i + 1))
done
kill -KILL -- "-$root" 2> /dev/null
for p in $tree; do kill -KILL "$p" 2> /dev/null; done
}
"$@" &
# The payload gets its PROCESS GROUP of its own (`set -m` makes every background job a group leader), because a
# process group is the one thing a descendant cannot escape: reparenting to init does not change it, and a group
# outlives its leader, so `kill -- -$child` still reaches the children AFTER the payload's root has gone. Without it
# the whole tree-kill is anchored on the root: a lane that starts a server and returns (`gpu-hold -- bash -c 'server &'`)
# leaves the root dead before the teardown runs, `descendants "$root"` is empty before it is even snapshotted, and the
# server outlives its ticket (engine#1649: measured, payload root exits by itself, `trap '' TERM` grandchild survives
# on main). `-m` also makes `$!` the payload's own pid, so `wait` still reports the payload's status.
set -m
# `< /dev/null` is NOT decoration: job control makes a background job keep the shell's stdin, so without it the payload
# would read whatever the CALLER is reading. Measured (bash 5.3.9), the fleet's own sweep shape
# `while read m; do gpu-hold -- bench "$m"; done < list`: without `set -m` the loop runs every entry, with `set -m` the
# first payload's `cat` swallows the rest and the sweep silently runs once. Background jobs on `main` already got
# /dev/null (a non-interactive shell's rule), so this keeps that semantic and takes only the process group.
"$@" < /dev/null &
child=$!
set +m # job control is needed only to place THIS job in its own group; leaving it on would make every later
# background job in this shell a background *group*, which is a semantic this script does not want to
# inherit. Measured: the payload keeps its own group after `set +m`, so the group kill still reaches it.
# shortcut: a payload that READS the controlling terminal is now a background group and would be stopped by SIGTTIN
# (job control's own rule). `gpu-hold` is for benchmarks and servers, which do not; an interactive payload on a tty is
# not a use this script supports, and the max-hold bound ends it (rc=124) rather than leaving the token held.
# The watchdog is a function so it can be given the command's pid: a subshell inherits a copy of the shell's
# variables, so a watchdog started before the command (the first version did) holds an empty pid and neither
# the abort nor the max hold stops anything.
Expand Down
107 changes: 107 additions & 0 deletions tests/gpu_hold_group_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The payload's tree must end even when the payload's ROOT is already gone when the teardown runs (engine#1649).
#
# The tree walk is anchored on the root: `descendants "$root"` after the root has exited is empty, so a snapshot of it
# is empty too and neither TERM nor KILL reaches anything. That is not a corner case here - it is what a lane does when
# it starts a server and returns (`gpu-hold -- bash -c 'server &'`), and it is how a 14 h llama-server held 51 GiB of
# GTT past its ticket. A process GROUP survives its leader, so the group kill covers exactly that shape.
#
# No device, no ticket, ~5 s: a private queue and token file, a `sleep` grandchild that ignores TERM, a payload that
# exits by itself. The pid this test signals is read from a file the payload wrote and is only ever signalled if
# /proc/<pid>/cmdline still carries this run's marker.
set -uo pipefail

hold=$(dirname "$0")/../scripts/gpu-hold
[ -x "$hold" ] || hold=$(dirname "$0")/../scripts/gpu-hold
[ -f "$hold" ] || { echo "skip: no scripts/gpu-hold"; exit 77; }
command -v ps > /dev/null 2>&1 || { echo "skip: no ps"; exit 77; }

scratch=$(mktemp -d)
marker="gpu-hold-group-$$-$RANDOM"
cleanup() {
p=$(cat "$scratch/pid" 2> /dev/null || true)
if [ -n "${p:-}" ] && kill -0 "$p" 2> /dev/null && tr '\0' ' ' < "/proc/$p/cmdline" 2> /dev/null | grep -q "$marker"; then
kill -9 "$p" 2> /dev/null
fi
rm -rf "$scratch"
}
trap cleanup EXIT
mkdir -p "$scratch/q"
cat > "$scratch/payload.sh" <<PAYLOAD
#!/bin/bash
( trap '' TERM; exec -a $marker sleep 300 ) &
echo \$! > "$scratch/pid"
sleep 1
exit 0
PAYLOAD
chmod +x "$scratch/payload.sh"

GPU_QUEUE_DIR=$scratch/q GPU_TOKEN_FILE=$scratch/token GPU_HOLD_LOG=$scratch/log \
GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=1 GPU_HOLD_KILL_GRACE=2 \
GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE=$scratch/tctl GPU_HOLD_THERMAL_FLAG=$scratch/none \
bash "$hold" -t 30 -l "$marker" -- "$scratch/payload.sh" > "$scratch/out" 2>&1
rc=$?

pid=$(cat "$scratch/pid" 2> /dev/null || true)
if [ -z "$pid" ]; then
echo "FAIL the payload never recorded its grandchild (see $scratch/out)"; sed 's/^/ /' "$scratch/out"; exit 1
fi
if kill -0 "$pid" 2> /dev/null && tr '\0' ' ' < "/proc/$pid/cmdline" 2> /dev/null | grep -q "$marker"; then
echo "FAIL a TERM-ignoring grandchild ($pid) outlived the hold after the payload's ROOT exited by itself -"
echo " the tree walk is anchored on a dead root; the group kill is what covers this shape (engine#1649)"
exit 1
fi
echo "ok the grandchild dies even though the payload's root had already exited (hold rc=$rc)"

# `set -m` must not change what the caller sees: the payload's OWN exit status still comes back through `wait`.
cat > "$scratch/status.sh" <<'STATUS'
#!/bin/bash
exit 7
STATUS
chmod +x "$scratch/status.sh"
GPU_QUEUE_DIR=$scratch/q GPU_TOKEN_FILE=$scratch/token GPU_HOLD_LOG=$scratch/log2 \
GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=1 GPU_HOLD_KILL_GRACE=2 \
GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE=$scratch/tctl GPU_HOLD_THERMAL_FLAG=$scratch/none \
bash "$hold" -t 30 -l "$marker-status" -- "$scratch/status.sh" > "$scratch/out2" 2>&1
src=$?
if [ "$src" = 7 ]; then
echo "ok a payload that exits 7 still reports 7 to the caller"
else
echo "FAIL a payload that exits 7 reported $src through gpu-hold (set -m must not change the status)"; exit 1
fi
# The payload must not read the CALLER's stdin: `set -m` makes a background job keep the shell's stdin, and the fleet's own
# sweep shape is `while read m; do gpu-hold -- bench "$m"; done < list` - without the payload's `< /dev/null` the first job
# swallows the rest of the list and the sweep silently runs once (measured, bash 5.3.9).
cat > "$scratch/eat.sh" <<'EAT'
#!/bin/bash
cat > /dev/null
EAT
chmod +x "$scratch/eat.sh"
consumed=$(printf 'a\nb\nc\n' | { n=0; while read -r m; do
n=$((n + 1))
GPU_QUEUE_DIR=$scratch/q GPU_TOKEN_FILE=$scratch/token GPU_HOLD_LOG=$scratch/log3 \
GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=1 GPU_HOLD_KILL_GRACE=2 \
GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE=$scratch/tctl GPU_HOLD_THERMAL_FLAG=$scratch/none \
bash "$hold" -t 30 -l "$marker-eat-$n" -- "$scratch/eat.sh" > /dev/null 2>&1
done; printf '%s' "$n"; })
if [ "$consumed" = 3 ]; then
echo "ok the payload cannot swallow the caller's stdin (a 3-entry sweep runs 3 times)"
else
echo "FAIL a sweep over 3 entries ran $consumed time(s): the payload is reading the caller's stdin"; exit 1
fi
echo "gpu_hold_group: ok"
Loading