Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmake/tests-gpu-hold.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,9 @@ add_test(NAME gpu_hold_group
# resolve under DIFFERENT bases, so two callers could each hold a token and both run GPU work. Asserts the two
# resolve under one base in all three states, plus that the script's two predicates are textually identical.
add_test(NAME gpu_hold_hold_dir COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_hold_dir_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold)

# The holder can PROVE it holds the ticket. `[ -e "$TOKEN" ]` is true whenever ANY lane holds the box, so a guard built
# on it passes exactly when starting work is most harmful - measured 2026-11-14, when a lane's runner served on the GPU
# for 40 s inside another lane's 45-minute hold. gpu-hold exports the exact token line it wrote (GPU_HOLD_TICKET) and the
# label, and this asserts the child sees both, on a sandboxed HOME.
add_test(NAME gpu_hold_child_identity COMMAND bash ${CMAKE_SOURCE_DIR}/tests/gpu_hold_child_identity_test.sh ${CMAKE_SOURCE_DIR}/scripts/gpu-hold)
8 changes: 7 additions & 1 deletion scripts/gpu-hold
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,13 @@ set -m
# `while read m; do gpu-hold -- bench "$m"; done < list`: without `set -m` the loop runs every entry, with `set -m` the
# first payload's `cat` swallows the rest and the sweep silently runs once. Background jobs on `main` already got
# /dev/null (a non-interactive shell's rule), so this keeps that semantic and takes only the process group.
"$@" < /dev/null &
# A child cannot otherwise prove it is the HOLDER rather than a bystander: `[ -e "$TOKEN" ]` is true whenever ANY lane
# holds the box, so a guard written with it passes exactly when starting work is most harmful (measured 2026-11-14: my
# own runner did that for 40 s against another lane's 45-minute ticket). The token's content is what this run wrote, so
# give it to the child - exact, not a string guess:
# GPU_HOLD_TICKET the exact line in the token file ("gpu-hold <label> pid <pid> <ts>")
# GPU_HOLD_LABEL the label, sanitised, as it appears in that line
env GPU_HOLD_TICKET="$mine" GPU_HOLD_LABEL="$SAFE" "$@" < /dev/null &
child=$!
set +m # job control is needed only to place THIS job in its own group; leaving it on would make every later
# background job in this shell a background *group*, which is a semantic this script does not want to
Expand Down
23 changes: 21 additions & 2 deletions scripts/npu-kernel-bitx.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,27 @@ if [ -z "$bin" ]; then
done
fi
[ -n "$bin" ] && [ -x "$bin" ] || { echo "npu-kernel-bitx: no 1bit binary (set ONEBIT_BIN=)" >&2; exit 2; }
if [ ! -e "$HOME/NPU-TOKEN" ]; then
echo "npu-kernel-bitx: not inside an npu-hold ticket (no ~/NPU-TOKEN); refusing to start a serve on the device" >&2
# OWNERSHIP, not presence. `[ -e ~/NPU-TOKEN ]` is true whenever ANY lane holds the device, so the old check let this
# script start a serve inside someone else's ticket while telling itself it was "inside an npu-hold" (engine#2434's
# family; measured 2026-11-14 on a sibling runner that did exactly that for 40 s). npu-hold/gpu-hold now hand the child
# the line they wrote into the token, so compare against that. A caller that holds the device in a shell created some
# other way can pass NPU_HOLD_TICKET explicitly; without it, refuse rather than guess.
_tok=${NPU_TOKEN_FILE:-$HOME/NPU-TOKEN}
# WHICH VARIABLE: the one the helper that actually runs this script exports. `npu-hold` execs `gpu-hold`, and gpu-hold hands
# its child GPU_HOLD_TICKET - nobody exports NPU_HOLD_TICKET. The first version of this check required NPU_HOLD_TICKET, so it
# refused EVERY run and told the reader "the helper exports NPU_HOLD_TICKET for you", which was false (engine#2447 review,
# measured: inside a real npu-hold the script refused with exactly that message). NPU_HOLD_TICKET stays as an override name
# for a caller that sets it by hand; GPU_HOLD_TICKET is the one that exists on the helper path.
_want=${NPU_HOLD_TICKET:-${GPU_HOLD_TICKET:-}}
if [ -z "$_want" ]; then
echo "npu-kernel-bitx: refusing: no ticket line in the environment (NPU_HOLD_TICKET or GPU_HOLD_TICKET), so this run" >&2
echo " cannot prove it owns the device. Run it under: npu-hold -l <label> -t <min> -- $0 ..." >&2
exit 2
fi
if [ ! -e "$_tok" ] || [ "$(cat "$_tok" 2>/dev/null)" != "$_want" ]; then
echo "npu-kernel-bitx: refusing: the device ticket is not ours." >&2
echo " ours: $_want" >&2
echo " in file: $(cat "$_tok" 2>/dev/null || echo '(absent)')" >&2
exit 2
fi
container=${1:-}; kernels=${2:-}
Expand Down
116 changes: 116 additions & 0 deletions tests/gpu_hold_child_identity_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# SPDX-License-Identifier: Apache-2.0
#
# gpu_hold_child_identity_test.sh - the holder can PROVE it holds the ticket, and a bystander cannot.
#
# Why: `[ -e ~/GPU-TOKEN ]` is true whenever ANY lane holds the box, so a guard written with it passes exactly when
# starting work is most harmful. Measured 2026-11-14: a lane's runner used that check, another lane held a 45-minute
# ticket, and the runner served on the GPU for 40 s inside someone else's hold. gpu-hold hands its child the exact line
# it wrote into the token (GPU_HOLD_TICKET) and the label (GPU_HOLD_LABEL); this test asserts both, on a sandboxed HOME so
# the real ~/GPU-QUEUE and ~/GPU-TOKEN are never touched.
#
# usage: gpu_hold_child_identity_test.sh <path-to-gpu-hold>
set -u
HOLD=${1:?usage: gpu_hold_child_identity_test.sh <path-to-gpu-hold>}
SRC=$(cd "$(dirname "$HOLD")/.." && pwd) # the tree, for scripts/npu-kernel-bitx.sh
tmp=$(mktemp -d "${TMPDIR:-/tmp}/ghid.XXXXXX") || exit 77
trap 'rm -rf "$tmp"' EXIT
fails=0

out=$(HOME="$tmp" ONEBIT_HOLD_DIR="$tmp/hold" GPU_HOLD_COOLDOWN=0 GPU_HOLD_RELEASE_WAIT=0 \
GPU_HOLD_WAIT=20 GPU_HOLD_MINUTES=1 GPU_HOLD_THERMAL_FLAG="$tmp/none" \
timeout 60 "$HOLD" -l child-identity -- \
bash -c 'printf "TICKET=%s\nLABEL=%s\nINFILE=%s\n" "${GPU_HOLD_TICKET:-}" "${GPU_HOLD_LABEL:-}" "$(cat "$1" 2>/dev/null)"' _ "$tmp/GPU-TOKEN" 2>&1)
rc=$?

ticket=$(printf '%s\n' "$out" | sed -n 's/^TICKET=//p' | head -1)
label=$(printf '%s\n' "$out" | sed -n 's/^LABEL=//p' | head -1)
infile=$(printf '%s\n' "$out" | sed -n 's/^INFILE=//p' | head -1)

printf ' rc=%d ticket=%s\n' "$rc" "${ticket:-<unset>}"
if [ "$rc" != 0 ]; then echo " FAIL the hold did not run (rc=$rc)"; exit 1; fi
if [ -z "$ticket" ]; then
echo " FAIL GPU_HOLD_TICKET is unset: a child cannot tell holding from being a bystander"
fails=$((fails + 1))
elif [ "$ticket" != "$infile" ]; then
echo " FAIL the exported ticket does not match the token file"
echo " exported: $ticket"
echo " in file: $infile"
fails=$((fails + 1))
else
echo " ok the child's GPU_HOLD_TICKET is exactly the line in the token"
fi
if [ "$label" != "child-identity" ]; then
echo " FAIL GPU_HOLD_LABEL is '$label', want 'child-identity'"
fails=$((fails + 1))
else
echo " ok GPU_HOLD_LABEL carries the sanitised label"
fi
# THE ARM THAT MATTERS, and the first version of this file did not have it: a token present but NOT ours must be refused by a
# real consumer. The old arm here ran `env -u X bash -c '[ -z "$X" ]'` - it unset the variables and then asserted they were
# unset, a tautology that printed ok unconditionally and pinned nothing (engine#2447 review). The incident this file exists for
# is "another lane holds the token and the guard still passes", so that is the case to drive.
foreign="gpu-hold another-lane pid 999999 2026-11-14T00:00:00-04:00"
printf '%s\n' "$foreign" > "$tmp/NPU-TOKEN"
out=$(ONEBIT_BIN=/bin/true NPU_TOKEN_FILE="$tmp/NPU-TOKEN" NPU_HOLD_TICKET= bash "$SRC/scripts/npu-kernel-bitx.sh" /x /y 2>&1)
rc=$?
if [ "$rc" = 2 ] && printf '%s' "$out" | grep -q 'cannot prove it owns the device'; then
echo " ok a foreign token with no ticket line is REFUSED by the consumer (rc 2)"
else
echo " FAIL the consumer did not refuse a foreign token: rc=$rc"
fails=$((fails + 1))
fi
out=$(ONEBIT_BIN=/bin/true NPU_TOKEN_FILE="$tmp/NPU-TOKEN" NPU_HOLD_TICKET="$foreign" \
bash "$SRC/scripts/npu-kernel-bitx.sh" /x /y 2>&1)
# engine#2447 review: this arm used to assert only the ABSENCE of the refusal text, so a run that died for any
# other reason (bad usage, missing binary, a reworded message) passed it. Assert POSITIVE progress instead: the
# script prints the device line only after the guard, so a refused run cannot produce it. Mutant: give the
# consumer a ticket line that does NOT match the token and this arm must fail.
if printf '%s' "$out" | grep -q '^device' && ! printf '%s' "$out" | grep -q 'cannot prove it owns the device'; then
echo " ok a matching ticket line gets past the consumer guard (reached the device line)"
else
echo " FAIL the consumer refused a ticket line that matches the token, or never got past the guard"
printf '%s\n' "$out" | sed -n '1,3p' | sed 's/^/ /'
fails=$((fails + 1))
fi
# engine#2447 review: the CONTENT comparison must be exercised by a NON-EMPTY line that differs. The arms above
# cover "no ticket line at all" (refused for absence) and "a line equal to the token" (accepted); neither reaches
# `[ "$(cat $_tok)" != "$_want" ]`, so dropping that comparison passed 0 failures. This is the arm for it: a
# well-formed ticket line for a DIFFERENT hold, with the token present.
other="gpu-hold some-other-lane pid 424242 2026-11-14T00:00:00-04:00"
out=$(ONEBIT_BIN=/bin/true NPU_TOKEN_FILE="$tmp/NPU-TOKEN" NPU_HOLD_TICKET="$other" \
bash "$SRC/scripts/npu-kernel-bitx.sh" /x /y 2>&1)
if printf '%s' "$out" | grep -q 'the device ticket is not ours'; then
echo " ok a NON-EMPTY ticket line that differs from the token is refused ('not ours')"
else
echo " FAIL a well-formed ticket line for a different hold was not refused as 'not ours'"
printf '%s\n' "$out" | sed -n '1,3p' | sed 's/^/ /'
fails=$((fails + 1))
fi
# and the variable the helper chain ACTUALLY exports must be the one accepted - the P0: the first version required
# NPU_HOLD_TICKET, which no helper sets, so every run was refused.
out=$(ONEBIT_BIN=/bin/true NPU_TOKEN_FILE="$tmp/NPU-TOKEN" GPU_HOLD_TICKET="$foreign" NPU_HOLD_TICKET= \
bash "$SRC/scripts/npu-kernel-bitx.sh" /x /y 2>&1)
if printf '%s' "$out" | grep -q '^device' && ! printf '%s' "$out" | grep -q 'cannot prove it owns the device'; then
echo " ok GPU_HOLD_TICKET (what the helper chain exports) is accepted (reached the device line)"
else
echo " FAIL GPU_HOLD_TICKET is not accepted although it is what npu-hold's chain exports"
printf '%s\n' "$out" | sed -n '1,3p' | sed 's/^/ /'
fails=$((fails + 1))
fi

echo "gpu_hold_child_identity: $fails failure(s)"
exit $((fails > 0))
Loading