From a54b42749d336a74198d1092d550cea1159eca54 Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 9 Nov 2026 19:13:00 -0400 Subject: [PATCH] gpu-hold: the queue and the token test the same thing, so they cannot resolve under different bases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `QUEUE` tested `-d "$HOLD/gpu-queue"` while `TOKEN` tested `-d "$HOLD"`. On a box where the fleet directory exists but `gpu-queue` does not — a partially-run install, or anything that creates `/var/lib/1bit-fleet` before the queue — the queue resolved to `~/GPU-QUEUE` and the token to `$HOLD/gpu-token`. Two callers in that window hold **different token files** and both run GPU work, which is the silent double-hold the fleet directory exists to prevent (engine #1504, #1609). Both lines now test `-d "$HOLD"`, and `$QUEUE` is created under it by the existing `mkdir -p` if the installer has not made it yet. Why this survived #1609's own test: `tests/gpu_hold_queue_test.sh` has **two** arms (fleet state fully installed, absent), and the resolution has **three** directory states. The missing one is the split. The new `tests/gpu_hold_hold_dir_test.sh` covers all three, under a temporary `HOME` so the real `~/GPU-QUEUE` and `~/GPU-TOKEN` are never touched, and additionally asserts the two predicates are textually identical. Fails on `origin/main`'s script, in two ways: $ bash tests/gpu_hold_hold_dir_test.sh /tmp/gpu-hold.main FAIL token is under …/fleet-dir-only/hold but no queue directory was created there - the two resolved apart FAIL QUEUE and TOKEN test different things after $HOLD (queue=[/gpu-queue] token=[]) - a split state is possible Passes with this commit, all three states: fleet-installed token=…/hold/gpu-token fleet-dir-only token=…/hold/gpu-token (the split state, now consistent) no-fleet-dir token=…/GPU-TOKEN Registered in `cmake/tests-gpu-hold.cmake` rather than inline, which is where #1699/#1742 put this family. Two things the test itself got wrong first, both fixed: its prep steps used the real `$HOME` (created `~/hold`, removed), and its first token-path parse kept the trailing timestamp. Refs #1609, #1504, #1696. --- cmake/tests-gpu-hold.cmake | 6 +++ scripts/gpu-hold | 10 +++- tests/gpu_hold_hold_dir_test.sh | 84 +++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100755 tests/gpu_hold_hold_dir_test.sh diff --git a/cmake/tests-gpu-hold.cmake b/cmake/tests-gpu-hold.cmake index b61cebcc3..32c4deb1e 100644 --- a/cmake/tests-gpu-hold.cmake +++ b/cmake/tests-gpu-hold.cmake @@ -51,3 +51,9 @@ add_test(NAME gpu_hold_queue COMMAND ${CMAKE_SOURCE_DIR}/tests/gpu_hold_queue_te add_test(NAME gpu_hold_survivors COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_survivors_test.sh) add_test(NAME gpu_hold_group COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_group_test.sh) +# engine#1609 follow-up: the resolution has THREE directory states and `gpu_hold_queue` above covers two of them +# (fleet state fully installed, and absent). The third -- the hold directory exists but gpu-queue does not, which is +# what a partially-run install or any `mkdir` of /var/lib/1bit-fleet leaves -- is where the queue and the token used to +# resolve under DIFFERENT bases, so two callers could each hold a token and both run GPU work. Asserts the two +# resolve under one base in all three states, plus that the script's two predicates are textually identical. +add_test(NAME gpu_hold_hold_dir COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_hold_dir_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold) diff --git a/scripts/gpu-hold b/scripts/gpu-hold index 339972229..e7a819737 100755 --- a/scripts/gpu-hold +++ b/scripts/gpu-hold @@ -65,8 +65,14 @@ MINUTES=${GPU_HOLD_MINUTES:-14} LABEL="" WAIT=${GPU_HOLD_WAIT:-5400} HOLD=${ONEBIT_HOLD_DIR:-/var/lib/1bit-fleet} -# the fleet's state wins when it is installed, and ~ is the fallback for a box that has not been set up that way -QUEUE=${GPU_QUEUE_DIR:-$([ -d "$HOLD/gpu-queue" ] && printf '%s' "$HOLD/gpu-queue" || printf '%s' "$HOME/GPU-QUEUE")} +# The fleet's state wins when it is installed, and ~ is the fallback for a box that has not been set up that way. +# BOTH lines must test the SAME thing. They used to differ -- the queue tested $HOLD/gpu-queue and the token tested +# $HOLD -- so on a box where /var/lib/1bit-fleet existed before gpu-queue did (a partially-run install, a mkdir for the +# NPU lock, anything that creates the directory first) the queue resolved to ~/GPU-QUEUE while the token resolved to +# $HOLD/gpu-token: two callers in that window hold DIFFERENT token files and both run GPU work, which is the silent +# double-hold the fleet directory exists to prevent. The directory being installed is the condition; the paths under it +# are then fixed, and $QUEUE is created by the mkdir below if the installer has not made it yet. +QUEUE=${GPU_QUEUE_DIR:-$([ -d "$HOLD" ] && printf '%s' "$HOLD/gpu-queue" || printf '%s' "$HOME/GPU-QUEUE")} TOKEN=${GPU_TOKEN_FILE:-$([ -d "$HOLD" ] && printf '%s' "$HOLD/gpu-token" || printf '%s' "$HOME/GPU-TOKEN")} RELEASE_WAIT=${GPU_HOLD_RELEASE_WAIT:-180} TCTL_START=${GPU_HOLD_TCTL_START:-75} diff --git a/tests/gpu_hold_hold_dir_test.sh b/tests/gpu_hold_hold_dir_test.sh new file mode 100755 index 000000000..fa2f6c2f8 --- /dev/null +++ b/tests/gpu_hold_hold_dir_test.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tests/gpu_hold_hold_dir_test.sh — the queue and the token must resolve under the SAME base. +# +# The fleet directory (`$ONEBIT_HOLD_DIR`, /var/lib/1bit-fleet) makes the GPU queue fleet-wide, so a runner under +# another account queues on the same token as an owner's tmux lane. If the two paths resolve under *different* bases -- +# which they did while the queue tested `$HOLD/gpu-queue` and the token tested `$HOLD` -- then on a box where the +# directory exists before `gpu-queue` does, two callers hold different token files and both run GPU work. That is the +# silent double-hold the directory exists to prevent, so it is asserted here rather than reasoned about. +# +# Everything happens under a temporary HOME, so the real ~/GPU-QUEUE and ~/GPU-TOKEN are never touched. +# +# -> 0 all three directory states agree, 1 a split was found, 77 cannot run here +set -u + +HOLD=${1:?usage: gpu_hold_hold_dir_test.sh } +[ -x "$HOLD" ] || { echo "not executable: $HOLD"; exit 1; } +command -v timeout > /dev/null || { echo "skip: no timeout"; exit 77; } + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fails=0 + +# One run per directory state. HOME is the temp dir, so both fallback paths land inside it. +run_state() { # + local name=$1 prep=$2 state="$tmp/$1" + mkdir -p "$state" + # $state, never $HOME: the prep runs before HOME is overridden, and using $HOME here would create directories in + # the real home (it did, on the first run of this test). + ( state="$state"; eval "$prep" ) + local out + out=$(HOME="$state" ONEBIT_HOLD_DIR="$state/hold" GPU_HOLD_COOLDOWN=0 GPU_HOLD_RELEASE_WAIT=0 \ + GPU_HOLD_WAIT=20 GPU_HOLD_MINUTES=1 GPU_HOLD_THERMAL_FLAG="$state/none" \ + timeout 60 "$HOLD" -l probe -- true 2>&1) + local rc=$? + # the token path is logged on take ("took ") and on release; the queue base is where the mkdir put it + local tok + tok=$(printf '%s\n' "$out" | sed -n 's/^gpu-hold\[[0-9]*\] \(took\|released\) \([^ ]*\).*/\2/p' | head -n 1) + printf ' %-22s rc=%d token=%s\n' "$name" "$rc" "${tok:-}" + if [ -z "$tok" ]; then + echo " FAIL no token path in the log - the run did not get far enough to test the resolution" + fails=$((fails + 1)) + return + fi + local tok_base; tok_base=$(dirname "$tok") + # the queue must have been created under the same base the token came from + if [ -d "$tok_base/gpu-queue" ] || [ -d "$tok_base/GPU-QUEUE" ]; then + : + else + echo " FAIL token is under $tok_base but no queue directory was created there - the two resolved apart" + fails=$((fails + 1)) + fi +} + +echo "=== queue and token resolve under one base" +run_state "fleet-installed" 'mkdir -p "$state/hold/gpu-queue"' +run_state "fleet-dir-only" 'mkdir -p "$state/hold"' +run_state "no-fleet-dir" ':' + +# and the two lines in the script must test the same thing, which is the invariant the states above observe +q_pred=$(grep -E '^QUEUE=' "$HOLD" | sed -n 's/.*\[ -d "\$HOLD\([^"]*\)" \].*/\1/p') +t_pred=$(grep -E '^TOKEN=' "$HOLD" | sed -n 's/.*\[ -d "\$HOLD\([^"]*\)" \].*/\1/p') +echo " predicates after \$HOLD: queue=[$q_pred] token=[$t_pred]" +if [ "$q_pred" != "$t_pred" ]; then + echo " FAIL QUEUE and TOKEN test different things after \$HOLD (queue=[$q_pred] token=[$t_pred]) - a split state is possible" + fails=$((fails + 1)) +fi + +[ "$fails" = 0 ] && echo "ok queue and token resolve under the same base in all three directory states" +exit "$fails"