Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ add_test(NAME release_check_row COMMAND ${CMAKE_SOURCE_DIR}/os/mini/release-chec
# of the ciphertext, Ed25519 over the manifest) and checks it; the test is a throwaway set in a temp
# dir, so no key, no device and no build: the four refusals are a set with no manifest, a file the
# manifest does not list, a ciphertext byte flipped after signing, and a manifest signed by another key.
# the handoff driver must not orphan its server on SIGKILL (PR_SET_PDEATHSIG); no GPU, no model
add_test(NAME kvshare_orphan COMMAND ${CMAKE_SOURCE_DIR}/tests/kvshare_orphan_test.sh)
add_test(NAME seal_manifest COMMAND ${CMAKE_SOURCE_DIR}/tests/seal_manifest_test.sh)
# engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as
# it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that
Expand Down
19 changes: 18 additions & 1 deletion tests/kvshare_fd_handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,22 @@
CTX = os.environ.get("CTX", "4096")


def die_with_parent():
"""Have the kernel kill the server if this process dies, however it dies.

The `finally` below reaps the server on a clean exit, a Ctrl-C and a SIGTERM, but not on SIGKILL (an OOM kill,
a CI timeout, `kill -9` by an operator): the child is in its own session, so it would survive and keep its HRX
context -- and on this box an orphaned llama-server held 51 GiB of GTT for 14 h and knocked out another
service. PR_SET_PDEATHSIG closes that case: the kernel signals the child when the parent dies, whatever the
reason. Linux only, which is where the HRX route runs; elsewhere the flag is simply not set.
"""
try:
import ctypes, signal
ctypes.CDLL("libc.so.6", use_errno=True).prctl(1, signal.SIGTERM) # PR_SET_PDEATHSIG
except Exception:
pass


def free_port():
s = socket.socket()
s.bind(("127.0.0.1", 0))
Expand Down Expand Up @@ -133,7 +149,8 @@ def say(msg):
srv = subprocess.Popen([SRV, "-m", MODEL, "--host", "127.0.0.1", "--port", str(port), "--device", "HRX0",
"-ngl", "99", "--jinja", "-c", CTX, "-np", "1", "-fa", "on", "--no-warmup",
"--slot-save-path", slotdir],
env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True)
env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True,
preexec_fn=die_with_parent)
try:
up, secs = wait_health(api, srv, out)
say(f"server pid {srv.pid} on :{port}, health up={up} after {secs:.1f}s")
Expand Down
66 changes: 66 additions & 0 deletions tests/kvshare_orphan_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The handoff driver must not be able to orphan its server. tests/kvshare_fd_handoff.py runs an HRX llama-server in
# its own session, so the `finally` that reaps it covers a clean exit, Ctrl-C and SIGTERM -- but not SIGKILL (OOM,
# a CI timeout, `kill -9`). An orphaned llama-server held 51 GiB of GTT for 14 h on this box on 2026-11-08 and
# knocked out another service, so the child is spawned with PR_SET_PDEATHSIG. This pins that: SIGKILL the parent
# and the child must go with it.
#
# No GPU and no model: the child is a sleep, spawned through the driver's own `die_with_parent`.
set -uo pipefail
here=$(cd "$(dirname "$0")" && pwd)
scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
fail=0
check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; }

python3 - "$here" > "$scratch/child.pid" <<'PY' &
import subprocess, sys, time
sys.path.insert(0, sys.argv[1])
import kvshare_fd_handoff as k
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"],
preexec_fn=k.die_with_parent, start_new_session=True)
print(child.pid, flush=True)
time.sleep(120)
PY
parent=$!
for _ in $(seq 1 50); do [ -s "$scratch/child.pid" ] && break; sleep 0.1; done
child=$(cat "$scratch/child.pid" 2>/dev/null || true)
check "the driver spawned a child in its own session (pid ${child:-none})" '[ -n "$child" ] && kill -0 "$child" 2>/dev/null'

kill -9 "$parent" 2>/dev/null
wait "$parent" 2>/dev/null
gone=0
for _ in $(seq 1 40); do kill -0 "$child" 2>/dev/null || { gone=1; break; }; sleep 0.05; done
check "SIGKILL of the driver takes the child with it (PR_SET_PDEATHSIG)" '[ "$gone" = 1 ]'

# and the negative control: a child spawned *without* it survives, which is what the flag is protecting against
python3 - "$here" > "$scratch/leak.pid" <<'PY' &
import subprocess, sys, time
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
print(child.pid, flush=True)
time.sleep(120)
PY
parent=$!
for _ in $(seq 1 50); do [ -s "$scratch/leak.pid" ] && break; sleep 0.1; done
leak=$(cat "$scratch/leak.pid" 2>/dev/null || true)
kill -9 "$parent" 2>/dev/null; wait "$parent" 2>/dev/null; sleep 0.3
check "without the flag the child is orphaned (the case this test exists for)" '[ -n "$leak" ] && kill -0 "$leak" 2>/dev/null'
kill -9 "$leak" 2>/dev/null

if [ $fail -ne 0 ]; then echo FAIL; exit 1; fi
echo PASS
Loading