From 0fbdb9340b8c0d1aac9760e8a0f62d9089ef5a09 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 06:37:42 -0400 Subject: [PATCH 1/2] gpu-hold: end the payload's PROCESS GROUP, so the tree dies even when its root is already gone (Fixes #1649) `kill_tree` walks `descendants "$root"`. When the payload's root has already exited - which is what a lane does when it starts a server and returns (`gpu-hold -- bash -c 'server &'`) - that walk is empty *before it is snapshotted*, so the TERM and KILL passes have nothing to signal. Measured on `origin/main` (`8ff2814e`) with a private queue and token file and a payload that starts a `trap '' TERM` grandchild and then exits by itself: `rc=0`, grandchild **ALIVE**. The three closed #1624 PRs' heads do the same; the snapshot fix is correct for the case it was written for and inert here. A process GROUP outlives its leader and reparenting does not change it, so two lines cover the shape: * `set -m` before launching the payload: every background job then leads its own group, and `$!` is still the payload's pid, so `wait` still returns the payload's own status (arm in the test asserts a payload that exits 7 reports 7); * `kill -TERM/--KILL -- "-$root"` in `kill_tree` before the walk. The walk stays as the fallback for a payload that moved itself into another group. The new `tests/gpu_hold_group_test.sh` fails on `origin/main` and passes here; `tests/gpu_hold_tree_test.sh` (the #1624 arm) stays green, so both shapes are covered. --- CMakeLists.txt | 2 + scripts/gpu-hold | 26 ++++++++++- tests/gpu_hold_group_test.sh | 87 ++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100755 tests/gpu_hold_group_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index addcb8f42..5df0f5454 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -277,6 +277,8 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test. # wedged in uninterruptible sleep, which user space cannot create, so this covers the other half: a holder that released with # a survivor must leave a record, and the NEXT holder must warn about it and clear it. add_test(NAME gpu_hold_survivors COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_survivors_test.sh) +add_test(NAME gpu_hold_group + COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_group_test.sh) # engine#1594: every target ci.yml's `cmake --build --target …` list names must exist in the configuration ci.yml # creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja # rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 35b674b89..339972229 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -313,16 +313,40 @@ kill_tree() { # ticket. The KILL pass then cannot see it and the orphan survives the hold -- which is the failure this function was # written to end. A snapshot also fixes the grace period: `kill -0` with a pid list is true while ANY of them lives, # so a wrapper that exits promptly no longer denies its children the grace they were promised. + # The group first: it is named by the payload's own pid and stays valid while ANY member lives, so this reaches + # descendants the walk cannot see (the root already gone, or a child that reparented before the snapshot). The + # walk stays as the fallback for a payload that moved itself into another group. + kill -TERM -- "-$root" 2> /dev/null local tree; tree=$(descendants "$root") for p in $tree; do kill -TERM "$p" 2> /dev/null; done while kill -0 $tree 2> /dev/null && [ "$i" -lt "$grace" ]; do sleep 1 i=$((i + 1)) done + kill -KILL -- "-$root" 2> /dev/null for p in $tree; do kill -KILL "$p" 2> /dev/null; done } -"$@" & +# The payload gets its PROCESS GROUP of its own (`set -m` makes every background job a group leader), because a +# process group is the one thing a descendant cannot escape: reparenting to init does not change it, and a group +# outlives its leader, so `kill -- -$child` still reaches the children AFTER the payload's root has gone. Without it +# the whole tree-kill is anchored on the root: a lane that starts a server and returns (`gpu-hold -- bash -c 'server &'`) +# leaves the root dead before the teardown runs, `descendants "$root"` is empty before it is even snapshotted, and the +# server outlives its ticket (engine#1649: measured, payload root exits by itself, `trap '' TERM` grandchild survives +# on main). `-m` also makes `$!` the payload's own pid, so `wait` still reports the payload's status. +set -m +# `< /dev/null` is NOT decoration: job control makes a background job keep the shell's stdin, so without it the payload +# would read whatever the CALLER is reading. Measured (bash 5.3.9), the fleet's own sweep shape +# `while read m; do gpu-hold -- bench "$m"; done < list`: without `set -m` the loop runs every entry, with `set -m` the +# first payload's `cat` swallows the rest and the sweep silently runs once. Background jobs on `main` already got +# /dev/null (a non-interactive shell's rule), so this keeps that semantic and takes only the process group. +"$@" < /dev/null & child=$! +set +m # job control is needed only to place THIS job in its own group; leaving it on would make every later + # background job in this shell a background *group*, which is a semantic this script does not want to + # inherit. Measured: the payload keeps its own group after `set +m`, so the group kill still reaches it. +# shortcut: a payload that READS the controlling terminal is now a background group and would be stopped by SIGTTIN +# (job control's own rule). `gpu-hold` is for benchmarks and servers, which do not; an interactive payload on a tty is +# not a use this script supports, and the max-hold bound ends it (rc=124) rather than leaving the token held. # The watchdog is a function so it can be given the command's pid: a subshell inherits a copy of the shell's # variables, so a watchdog started before the command (the first version did) holds an empty pid and neither # the abort nor the max hold stops anything. diff --git a/tests/gpu_hold_group_test.sh b/tests/gpu_hold_group_test.sh new file mode 100755 index 000000000..8fae5ec8b --- /dev/null +++ b/tests/gpu_hold_group_test.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The payload's tree must end even when the payload's ROOT is already gone when the teardown runs (engine#1649). +# +# The tree walk is anchored on the root: `descendants "$root"` after the root has exited is empty, so a snapshot of it +# is empty too and neither TERM nor KILL reaches anything. That is not a corner case here - it is what a lane does when +# it starts a server and returns (`gpu-hold -- bash -c 'server &'`), and it is how a 14 h llama-server held 51 GiB of +# GTT past its ticket. A process GROUP survives its leader, so the group kill covers exactly that shape. +# +# No device, no ticket, ~5 s: a private queue and token file, a `sleep` grandchild that ignores TERM, a payload that +# exits by itself. The pid this test signals is read from a file the payload wrote and is only ever signalled if +# /proc//cmdline still carries this run's marker. +set -uo pipefail + +hold=$(dirname "$0")/../scripts/gpu-hold +[ -x "$hold" ] || hold=$(dirname "$0")/../scripts/gpu-hold +[ -f "$hold" ] || { echo "skip: no scripts/gpu-hold"; exit 77; } +command -v ps > /dev/null 2>&1 || { echo "skip: no ps"; exit 77; } + +scratch=$(mktemp -d) +marker="gpu-hold-group-$$-$RANDOM" +cleanup() { + p=$(cat "$scratch/pid" 2> /dev/null || true) + if [ -n "${p:-}" ] && kill -0 "$p" 2> /dev/null && tr '\0' ' ' < "/proc/$p/cmdline" 2> /dev/null | grep -q "$marker"; then + kill -9 "$p" 2> /dev/null + fi + rm -rf "$scratch" +} +trap cleanup EXIT +mkdir -p "$scratch/q" +cat > "$scratch/payload.sh" < "$scratch/pid" +sleep 1 +exit 0 +PAYLOAD +chmod +x "$scratch/payload.sh" + +GPU_QUEUE_DIR=$scratch/q GPU_TOKEN_FILE=$scratch/token GPU_HOLD_LOG=$scratch/log \ +GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=1 GPU_HOLD_KILL_GRACE=2 \ +GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE=$scratch/tctl GPU_HOLD_THERMAL_FLAG=$scratch/none \ + bash "$hold" -t 30 -l "$marker" -- "$scratch/payload.sh" > "$scratch/out" 2>&1 +rc=$? + +pid=$(cat "$scratch/pid" 2> /dev/null || true) +if [ -z "$pid" ]; then + echo "FAIL the payload never recorded its grandchild (see $scratch/out)"; sed 's/^/ /' "$scratch/out"; exit 1 +fi +if kill -0 "$pid" 2> /dev/null && tr '\0' ' ' < "/proc/$pid/cmdline" 2> /dev/null | grep -q "$marker"; then + echo "FAIL a TERM-ignoring grandchild ($pid) outlived the hold after the payload's ROOT exited by itself -" + echo " the tree walk is anchored on a dead root; the group kill is what covers this shape (engine#1649)" + exit 1 +fi +echo "ok the grandchild dies even though the payload's root had already exited (hold rc=$rc)" + +# `set -m` must not change what the caller sees: the payload's OWN exit status still comes back through `wait`. +cat > "$scratch/status.sh" <<'STATUS' +#!/bin/bash +exit 7 +STATUS +chmod +x "$scratch/status.sh" +GPU_QUEUE_DIR=$scratch/q GPU_TOKEN_FILE=$scratch/token GPU_HOLD_LOG=$scratch/log2 \ +GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=1 GPU_HOLD_KILL_GRACE=2 \ +GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE=$scratch/tctl GPU_HOLD_THERMAL_FLAG=$scratch/none \ + bash "$hold" -t 30 -l "$marker-status" -- "$scratch/status.sh" > "$scratch/out2" 2>&1 +src=$? +if [ "$src" = 7 ]; then + echo "ok a payload that exits 7 still reports 7 to the caller" +else + echo "FAIL a payload that exits 7 reported $src through gpu-hold (set -m must not change the status)"; exit 1 +fi +echo "gpu_hold_group: ok" From fa303e7530dcaea3059c8d4de794daa7accb978b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 07:42:10 -0400 Subject: [PATCH 2/2] gpu-hold: the payload keeps main's stdin (`< /dev/null`), and a test arm pins it (#1649 review) Review finding, reproduced on bash 5.3.9: **job control makes a background job keep the shell's stdin**, so under `set -m` a payload reads whatever the CALLER is reading. The fleet's own sweep shape `while read m; do gpu-hold -- bench "$m"; done < list` then runs ONCE, because the first payload's `cat` swallows the rest of the list: no set -m: [a] [b] [c] set -m: [a] b c `"$@" < /dev/null &` restores main's semantics exactly (a background job of a non-interactive shell already got /dev/null) and takes only the process group from the change. The new arm runs that sweep shape and asserts all three entries are consumed: it passes here and FAILS on the same script with the redirect removed (`ran 1 time(s)`). (`set +m` right after the launch was already in 1f1f1c0c, so the watchdog and any later background job are not affected.) --- tests/gpu_hold_group_test.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/gpu_hold_group_test.sh b/tests/gpu_hold_group_test.sh index 8fae5ec8b..30182ef84 100755 --- a/tests/gpu_hold_group_test.sh +++ b/tests/gpu_hold_group_test.sh @@ -84,4 +84,24 @@ if [ "$src" = 7 ]; then else echo "FAIL a payload that exits 7 reported $src through gpu-hold (set -m must not change the status)"; exit 1 fi +# The payload must not read the CALLER's stdin: `set -m` makes a background job keep the shell's stdin, and the fleet's own +# sweep shape is `while read m; do gpu-hold -- bench "$m"; done < list` - without the payload's `< /dev/null` the first job +# swallows the rest of the list and the sweep silently runs once (measured, bash 5.3.9). +cat > "$scratch/eat.sh" <<'EAT' +#!/bin/bash +cat > /dev/null +EAT +chmod +x "$scratch/eat.sh" +consumed=$(printf 'a\nb\nc\n' | { n=0; while read -r m; do + n=$((n + 1)) + GPU_QUEUE_DIR=$scratch/q GPU_TOKEN_FILE=$scratch/token GPU_HOLD_LOG=$scratch/log3 \ + GPU_HOLD_POLL=1 GPU_HOLD_WAIT=5 GPU_HOLD_RELEASE_WAIT=1 GPU_HOLD_KILL_GRACE=2 \ + GPU_HOLD_TCTL_START=100 GPU_HOLD_TCTL_FILE=$scratch/tctl GPU_HOLD_THERMAL_FLAG=$scratch/none \ + bash "$hold" -t 30 -l "$marker-eat-$n" -- "$scratch/eat.sh" > /dev/null 2>&1 +done; printf '%s' "$n"; }) +if [ "$consumed" = 3 ]; then + echo "ok the payload cannot swallow the caller's stdin (a 3-entry sweep runs 3 times)" +else + echo "FAIL a sweep over 3 entries ran $consumed time(s): the payload is reading the caller's stdin"; exit 1 +fi echo "gpu_hold_group: ok"