Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmake/tests-gpu-hold.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,9 @@ add_test(NAME gpu_hold_queue COMMAND ${CMAKE_SOURCE_DIR}/tests/gpu_hold_queue_te
add_test(NAME gpu_hold_survivors COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_survivors_test.sh)
add_test(NAME gpu_hold_group
COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_group_test.sh)
# engine#1609 follow-up: the resolution has THREE directory states and `gpu_hold_queue` above covers two of them
# (fleet state fully installed, and absent). The third -- the hold directory exists but gpu-queue does not, which is
# what a partially-run install or any `mkdir` of /var/lib/1bit-fleet leaves -- is where the queue and the token used to
# resolve under DIFFERENT bases, so two callers could each hold a token and both run GPU work. Asserts the two
# resolve under one base in all three states, plus that the script's two predicates are textually identical.
add_test(NAME gpu_hold_hold_dir COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_hold_dir_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold)
10 changes: 8 additions & 2 deletions scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,14 @@ MINUTES=${GPU_HOLD_MINUTES:-14}
LABEL=""
WAIT=${GPU_HOLD_WAIT:-5400}
HOLD=${ONEBIT_HOLD_DIR:-/var/lib/1bit-fleet}
# the fleet's state wins when it is installed, and ~ is the fallback for a box that has not been set up that way
QUEUE=${GPU_QUEUE_DIR:-$([ -d "$HOLD/gpu-queue" ] && printf '%s' "$HOLD/gpu-queue" || printf '%s' "$HOME/GPU-QUEUE")}
# The fleet's state wins when it is installed, and ~ is the fallback for a box that has not been set up that way.
# BOTH lines must test the SAME thing. They used to differ -- the queue tested $HOLD/gpu-queue and the token tested
# $HOLD -- so on a box where /var/lib/1bit-fleet existed before gpu-queue did (a partially-run install, a mkdir for the
# NPU lock, anything that creates the directory first) the queue resolved to ~/GPU-QUEUE while the token resolved to
# $HOLD/gpu-token: two callers in that window hold DIFFERENT token files and both run GPU work, which is the silent
# double-hold the fleet directory exists to prevent. The directory being installed is the condition; the paths under it
# are then fixed, and $QUEUE is created by the mkdir below if the installer has not made it yet.
QUEUE=${GPU_QUEUE_DIR:-$([ -d "$HOLD" ] && printf '%s' "$HOLD/gpu-queue" || printf '%s' "$HOME/GPU-QUEUE")}
TOKEN=${GPU_TOKEN_FILE:-$([ -d "$HOLD" ] && printf '%s' "$HOLD/gpu-token" || printf '%s' "$HOME/GPU-TOKEN")}
RELEASE_WAIT=${GPU_HOLD_RELEASE_WAIT:-180}
TCTL_START=${GPU_HOLD_TCTL_START:-75}
Expand Down
84 changes: 84 additions & 0 deletions tests/gpu_hold_hold_dir_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# tests/gpu_hold_hold_dir_test.sh — the queue and the token must resolve under the SAME base.
#
# The fleet directory (`$ONEBIT_HOLD_DIR`, /var/lib/1bit-fleet) makes the GPU queue fleet-wide, so a runner under
# another account queues on the same token as an owner's tmux lane. If the two paths resolve under *different* bases --
# which they did while the queue tested `$HOLD/gpu-queue` and the token tested `$HOLD` -- then on a box where the
# directory exists before `gpu-queue` does, two callers hold different token files and both run GPU work. That is the
# silent double-hold the directory exists to prevent, so it is asserted here rather than reasoned about.
#
# Everything happens under a temporary HOME, so the real ~/GPU-QUEUE and ~/GPU-TOKEN are never touched.
#
# <gpu-hold> -> 0 all three directory states agree, 1 a split was found, 77 cannot run here
set -u

HOLD=${1:?usage: gpu_hold_hold_dir_test.sh <path-to-gpu-hold>}
[ -x "$HOLD" ] || { echo "not executable: $HOLD"; exit 1; }
command -v timeout > /dev/null || { echo "skip: no timeout"; exit 77; }

tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
fails=0

# One run per directory state. HOME is the temp dir, so both fallback paths land inside it.
run_state() { # <state-name> <prepare-cmd, uses $state as the temp home>
local name=$1 prep=$2 state="$tmp/$1"
mkdir -p "$state"
# $state, never $HOME: the prep runs before HOME is overridden, and using $HOME here would create directories in
# the real home (it did, on the first run of this test).
( state="$state"; eval "$prep" )
local out
out=$(HOME="$state" ONEBIT_HOLD_DIR="$state/hold" GPU_HOLD_COOLDOWN=0 GPU_HOLD_RELEASE_WAIT=0 \
GPU_HOLD_WAIT=20 GPU_HOLD_MINUTES=1 GPU_HOLD_THERMAL_FLAG="$state/none" \
timeout 60 "$HOLD" -l probe -- true 2>&1)
local rc=$?
# the token path is logged on take ("took <path>") and on release; the queue base is where the mkdir put it
local tok
tok=$(printf '%s\n' "$out" | sed -n 's/^gpu-hold\[[0-9]*\] \(took\|released\) \([^ ]*\).*/\2/p' | head -n 1)
printf ' %-22s rc=%d token=%s\n' "$name" "$rc" "${tok:-<none>}"
if [ -z "$tok" ]; then
echo " FAIL no token path in the log - the run did not get far enough to test the resolution"
fails=$((fails + 1))
return
fi
local tok_base; tok_base=$(dirname "$tok")
# the queue must have been created under the same base the token came from
if [ -d "$tok_base/gpu-queue" ] || [ -d "$tok_base/GPU-QUEUE" ]; then
:
else
echo " FAIL token is under $tok_base but no queue directory was created there - the two resolved apart"
fails=$((fails + 1))
fi
}

echo "=== queue and token resolve under one base"
run_state "fleet-installed" 'mkdir -p "$state/hold/gpu-queue"'
run_state "fleet-dir-only" 'mkdir -p "$state/hold"'
run_state "no-fleet-dir" ':'

# and the two lines in the script must test the same thing, which is the invariant the states above observe
q_pred=$(grep -E '^QUEUE=' "$HOLD" | sed -n 's/.*\[ -d "\$HOLD\([^"]*\)" \].*/\1/p')
t_pred=$(grep -E '^TOKEN=' "$HOLD" | sed -n 's/.*\[ -d "\$HOLD\([^"]*\)" \].*/\1/p')
echo " predicates after \$HOLD: queue=[$q_pred] token=[$t_pred]"
if [ "$q_pred" != "$t_pred" ]; then
echo " FAIL QUEUE and TOKEN test different things after \$HOLD (queue=[$q_pred] token=[$t_pred]) - a split state is possible"
fails=$((fails + 1))
fi

[ "$fails" = 0 ] && echo "ok queue and token resolve under the same base in all three directory states"
exit "$fails"
Loading