Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,9 @@ add_test(NAME gpu_hold_mem_abort COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold
# Drives scripts/gpu-hold on a private queue: the ticket is deleted mid-wait, and separately rewritten to a dead pid so a
# second real gpu-hold reaps it and leaves the note the victim quotes. No device, no build; skips (77) without the script.
add_test(NAME gpu_hold_ticket_lost COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_ticket_lost_test.sh)
# engine#1701: the NPU hold's queue/token follow ONEBIT_HOLD_DIR, and its queue settings never reach the payload.
# Shell only (a stub gpu-hold plus the tree's own), no device, ~1 s.
add_test(NAME npu_hold COMMAND bash ${CMAKE_SOURCE_DIR}/tests/npu_hold_test.sh)
# engine#1485: the two port read-backs the route tests use (tests/serve_port.sh). Script-based, no device, no model --
# it feeds the parsers the exact lines `1bit serve` and `llama-server` print, because a read-back that silently returns
# empty turns into "the server did not come up" and is blamed on whatever the row under test checks.
Expand Down
90 changes: 90 additions & 0 deletions scripts/npu-hold
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# npu-hold [-t MINUTES] [-l LABEL] [-w WAIT_SECONDS] -- CMD [ARGS...]
#
# The NPU array's fair FIFO claim: gpu-hold's mechanism on the NPU's own queue/token. A lane that needs the array
# runs its command under npu-hold and BLOCKS until its turn. The owner's Bonsai serve is managed by
# npu-serve-keeper, which holds the token while the serve is up and stops it whenever a ticket is waiting.
# Thermal gates are relaxed (the NPU does not heat the die the way the GPU does); max hold default 20 minutes.
#
# Two things this file exists to fix (engine#1701), both of them drift rather than logic:
#
# * the queue, the token and the log follow the fleet directory exactly as scripts/gpu-hold does
# (engine#1504): HOLD is $ONEBIT_HOLD_DIR or /var/lib/1bit-fleet, and $HOLD/npu-queue IS the queue when that
# subdirectory exists - the same per-subdirectory test gpu-hold applies to $HOLD/gpu-queue, so the two holds
# can never disagree about where the fleet's state is. Without it the paths stay $HOME/NPU-*, which keeps a
# box that has not been set up unchanged.
# A shared queue is REFUSED (exit 2) while ~/.local/bin/npu-serve-keeper still reads $HOME/NPU-QUEUE: the
# keeper yields the array when a ticket appears in the queue IT reads, so a hold it cannot see would believe
# the device is free while the serve keeps it. Point the keeper at the same paths first, then set
# NPU_HOLD_SHARED=1 to confirm you have checked.
# * the hold is taken by THIS TREE's scripts/gpu-hold first ($ONEBIT_GPU_HOLD overrides), with the
# installer-managed ~/.local/bin copy as the fallback for a machine without a checkout, and a note printed
# when the two differ. The private copy drifted for three days on the survivor bound alone (engine#1639 /
# #1646); "fixed in the tree" has to mean fixed for NPU work too.
#
# The queue/token settings are given to gpu-hold ONLY, never to CMD: a gpu-hold inside CMD (the hw-gate's own
# GPU claim, a bench wrapper) must queue on the GPU, not on the NPU queue its parent already holds (that
# deadlocked the c18 gate on 2026-10-31). Everything after -- runs with those variables unset.
set -u
# Where the fleet's hold state lives. The discovery is deliberately IDENTICAL to scripts/gpu-hold's
# (engine#1504), including the default: the two holds must never disagree about where the state is, or the
# owner's "create /var/lib/1bit-fleet" step moves one of them and not the other.
HOLD=${ONEBIT_HOLD_DIR:-/var/lib/1bit-fleet}
if [ -d "$HOLD/npu-queue" ]; then
QUEUE=$HOLD/npu-queue; TOKEN=$HOLD/npu-token; LOG=$HOLD/npu-queue/.log; SHARED=1
else
QUEUE=$HOME/NPU-QUEUE; TOKEN=$HOME/NPU-TOKEN; LOG=$HOME/NPU-QUEUE/.log; SHARED=0
fi
# A shared queue is only safe once npu-serve-keeper watches it too (engine#1701): the keeper yields the NPU
# when a ticket appears in the queue IT reads, and hard-codes $HOME/NPU-QUEUE. A hold queued somewhere the
# keeper cannot see would believe the device is free while the serve keeps it -- two owners on the array,
# which is worse than the per-account drift this file fixes. So a shared queue is refused until the keeper's
# own path follows it, unless the caller says it has checked (NPU_HOLD_SHARED=1).
if [ "$SHARED" = 1 ] && [ "${NPU_HOLD_SHARED:-0}" != 1 ]; then
keeper=$HOME/.local/bin/npu-serve-keeper
if [ -f "$keeper" ] && grep -q 'NPU-QUEUE' "$keeper" 2> /dev/null; then
echo "npu-hold: REFUSED: $QUEUE is a shared NPU queue, but $(printf '%s' "$keeper") still reads \$HOME/NPU-QUEUE" >&2
echo "npu-hold: so the keeper would not see this ticket and could keep the array. Point the keeper at $QUEUE" >&2
echo "npu-hold: first, or set NPU_HOLD_SHARED=1 once you have checked that it does." >&2
exit 2
fi
fi
# The hold is taken by the tree's gpu-hold first, so "fixed in the tree" is fixed for NPU work as well; the
# installer-managed copy under ~/.local/bin is the fallback for a machine without a checkout (engine#1701).
here=$(cd "$(dirname "$0")" && pwd)
priv=$HOME/.local/bin/gpu-hold
if [ -n "${ONEBIT_GPU_HOLD:-}" ]; then GH=$ONEBIT_GPU_HOLD
elif [ -x "$here/gpu-hold" ]; then GH=$here/gpu-hold
elif [ -x "$priv" ]; then GH=$priv
else
echo "npu-hold: no gpu-hold found (looked at \$ONEBIT_GPU_HOLD, $here/gpu-hold, $priv)" >&2
exit 2
fi
if [ -x "$priv" ] && [ -x "$here/gpu-hold" ] && ! cmp -s "$priv" "$here/gpu-hold"; then
echo "npu-hold: note: $priv differs from the tree's $here/gpu-hold; using the tree's (ONEBIT_GPU_HOLD overrides)" >&2
fi
opts=(); while [ $# -gt 0 ] && [ "$1" != -- ]; do opts+=("$1"); shift; done
[ "${1:-}" = -- ] && shift
[ $# -gt 0 ] || { echo "npu-hold: usage: npu-hold [-t MIN] [-l LABEL] [-w SEC] -- CMD [ARGS...]" >&2; exit 2; }
exec env GPU_QUEUE_DIR="$QUEUE" GPU_TOKEN_FILE="$TOKEN" \
GPU_HOLD_MINUTES="${NPU_HOLD_MINUTES:-20}" GPU_HOLD_WAIT="${NPU_HOLD_WAIT:-7200}" \
GPU_HOLD_TCTL_START=90 GPU_HOLD_RELEASE_WAIT=0 GPU_HOLD_COOLDOWN=0 GPU_HOLD_IGNORE_THERMAL_HOLD=1 \
GPU_HOLD_LOG="$LOG" "$GH" "${opts[@]}" -- \
env -u GPU_QUEUE_DIR -u GPU_TOKEN_FILE -u GPU_HOLD_MINUTES -u GPU_HOLD_WAIT -u GPU_HOLD_TCTL_START \
-u GPU_HOLD_RELEASE_WAIT -u GPU_HOLD_COOLDOWN -u GPU_HOLD_IGNORE_THERMAL_HOLD -u GPU_HOLD_LOG "$@"
106 changes: 106 additions & 0 deletions tests/npu_hold_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# tests/npu_hold_test.sh -- scripts/npu-hold puts its queue/token/log where ONEBIT_HOLD_DIR says, and hands the
# queue settings to gpu-hold only (engine#1701).
#
# Hermetic: a stub gpu-hold records the environment it was given and runs the payload, so nothing here touches a
# real queue, a token or a device.
here=$(cd "$(dirname "$0")" && pwd)
root=$(cd "$here/.." && pwd)
fail=0
ck() { if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got '$2', wanted '$3'"; fail=1; fi; }

t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
# a stub that proves what npu-hold handed it, then runs the payload the same way the real one does
cat > "$t/gpu-hold" <<'STUB'
#!/usr/bin/env bash
printf '%s\n' "$GPU_QUEUE_DIR|$GPU_TOKEN_FILE|$GPU_HOLD_LOG|$GPU_HOLD_MINUTES|$GPU_HOLD_TCTL_START" > "$STUB_RECORD"
while [ $# -gt 0 ] && [ "$1" != -- ]; do shift; done
[ "${1:-}" = -- ] && shift
exec "$@"
STUB
chmod +x "$t/gpu-hold"

# 1. the fleet queue directory exists: it wins -- the same per-subdirectory rule scripts/gpu-hold uses for
# $HOLD/gpu-queue, so the two holds cannot disagree about where the state is
mkdir -p "$t/fleet/npu-queue"
env ONEBIT_HOLD_DIR="$t/fleet" ONEBIT_GPU_HOLD="$t/gpu-hold" STUB_RECORD="$t/r1" HOME="$t/home" \
"$root/scripts/npu-hold" -t 1 -l probe -- sh -c 'echo payload-ran' > "$t/out1" 2>&1
ck "the payload ran" "$(cat "$t/out1")" "payload-ran"
IFS='|' read -r q tk lg mins start < "$t/r1"
ck "queue follows ONEBIT_HOLD_DIR" "$q" "$t/fleet/npu-queue"
ck "token follows ONEBIT_HOLD_DIR" "$tk" "$t/fleet/npu-token"
ck "log follows ONEBIT_HOLD_DIR" "$lg" "$t/fleet/npu-queue/.log"
ck "the NPU hold still relaxes the thermal gate" "$start" "90"

# 2. no ONEBIT_HOLD_DIR: unchanged, $HOME/NPU-* as before
mkdir -p "$t/home"
env -u ONEBIT_HOLD_DIR ONEBIT_GPU_HOLD="$t/gpu-hold" STUB_RECORD="$t/r2" HOME="$t/home" \
"$root/scripts/npu-hold" -- sh -c ':' > /dev/null 2>&1
IFS='|' read -r q tk lg _ _ < "$t/r2"
ck "without ONEBIT_HOLD_DIR the queue is \$HOME/NPU-QUEUE" "$q" "$t/home/NPU-QUEUE"
ck "without ONEBIT_HOLD_DIR the token is \$HOME/NPU-TOKEN" "$tk" "$t/home/NPU-TOKEN"

# 3. the queue settings must NOT reach the payload: a gpu-hold inside it has to queue on the GPU
cat > "$t/payload" <<'P'
#!/usr/bin/env bash
echo "${GPU_QUEUE_DIR:-unset}|${GPU_TOKEN_FILE:-unset}|${GPU_HOLD_LOG:-unset}" > "$PAYLOAD_RECORD"
P
chmod +x "$t/payload"
# NPU_HOLD_SHARED=1 because this box's real ~/.local/bin/npu-serve-keeper still hard-codes $HOME/NPU-QUEUE,
# which is exactly what the guard above refuses -- the arm is about the env the payload gets, not the queue path.
env ONEBIT_HOLD_DIR="$t/fleet" NPU_HOLD_SHARED=1 ONEBIT_GPU_HOLD="$t/gpu-hold" STUB_RECORD="$t/r3" \
PAYLOAD_RECORD="$t/p3" "$root/scripts/npu-hold" -- "$t/payload" > /dev/null 2>&1
ck "the payload sees no NPU queue settings" "$(cat "$t/p3")" "unset|unset|unset"

# 3b. a shared queue is refused while npu-serve-keeper still reads $HOME/NPU-QUEUE (it would not see the ticket)
mkdir -p "$t/home2/.local/bin"
printf '#!/usr/bin/env bash\nQ=$HOME/NPU-QUEUE; T=$HOME/NPU-TOKEN\n' > "$t/home2/.local/bin/npu-serve-keeper"
chmod +x "$t/home2/.local/bin/npu-serve-keeper"
env ONEBIT_HOLD_DIR="$t/fleet" ONEBIT_GPU_HOLD="$t/gpu-hold" STUB_RECORD="$t/r4" HOME="$t/home2" \
"$root/scripts/npu-hold" -- sh -c ':' > "$t/out4b" 2>&1
ck "a shared queue is refused while the keeper reads \$HOME/NPU-QUEUE" "$?" "2"
ck "and the refusal says why" "$(grep -c 'still reads' "$t/out4b")" "1"
# ...and it is allowed once the caller confirms the keeper follows it
env ONEBIT_HOLD_DIR="$t/fleet" NPU_HOLD_SHARED=1 ONEBIT_GPU_HOLD="$t/gpu-hold" STUB_RECORD="$t/r5" HOME="$t/home2" \
"$root/scripts/npu-hold" -- sh -c ':' > /dev/null 2>&1
ck "NPU_HOLD_SHARED=1 allows it" "$([ -f "$t/r5" ] && echo recorded || echo missing)" "recorded"

# 4. integration: the tree's own gpu-hold takes the NPU queue when it is the only one present, and releases it.
# No device is touched - gpu-hold only creates a ticket/token and runs the payload - so this is hermetic too.
mkdir -p "$t/fleet2"
env ONEBIT_HOLD_DIR="$t/fleet2" ONEBIT_GPU_HOLD="$root/scripts/gpu-hold" HOME="$t/home" \
"$root/scripts/npu-hold" -t 1 -l itest -- sh -c 'echo integrated' > "$t/out4" 2>&1
ck "the tree gpu-hold ran the payload" "$(grep -c integrated "$t/out4")" "1"
ck "and released the NPU token" "$([ -f "$t/fleet2/npu-token" ] && echo present || echo gone)" "gone"
ck "with no ticket left behind" "$(ls -A "$t/fleet2/npu-queue" 2>/dev/null | grep -v '^\.' | wc -l)" "0"

# 4b. the tree's gpu-hold wins over a differing ~/.local/bin copy, and the difference is reported
mkdir -p "$t/home3/.local/bin"
printf '#!/usr/bin/env bash\n# stale private copy\nexit 0\n' > "$t/home3/.local/bin/gpu-hold"
chmod +x "$t/home3/.local/bin/gpu-hold"
env ONEBIT_HOLD_DIR="$t/fleet" STUB_RECORD="$t/r6" HOME="$t/home3" \
"$root/scripts/npu-hold" -- sh -c 'echo from-tree' > "$t/out4c" 2>&1
ck "the tree gpu-hold ran, not the stale private copy" "$(grep -c from-tree "$t/out4c")" "1"
ck "and the stale copy was reported" "$(grep -c 'differs from the tree' "$t/out4c")" "1"

# 5. a missing command is a usage error, not a silent success
env ONEBIT_GPU_HOLD="$t/gpu-hold" "$root/scripts/npu-hold" -- > /dev/null 2>&1
ck "no command is a usage error" "$?" "2"

[ "$fail" = 0 ] && echo "npu_hold: ok" || echo "npu_hold: FAILED"
exit "$fail"
Loading