From d76d6b9110f6d4b804da2fbf936e25f042368c1b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 14:02:24 -0400 Subject: [PATCH] tests: the handoff driver cannot orphan its server on SIGKILL (PR_SET_PDEATHSIG) tests/kvshare_fd_handoff.py runs an HRX llama-server in its own session and reaps it in a `finally`, which covers a clean exit, Ctrl-C and SIGTERM -- but not SIGKILL (an OOM kill, a CI timeout, `kill -9`). On this box an orphaned llama-server held 51 GiB of GTT for 14 h and knocked out another service, so a test that can leak a decoder holding an HRX context is exactly the wrong kind of test to have. The server is now spawned with PR_SET_PDEATHSIG (SIGTERM), so the kernel takes it down when this process dies whatever the reason. Linux only, which is where the HRX route runs; elsewhere the flag is simply not set. tests/kvshare_orphan_test.sh pins it without a GPU or a model: it spawns a sleep through the driver's own `die_with_parent`, SIGKILLs the parent and asserts the child is gone -- and, as the negative control, that a child spawned without the flag survives the same kill, which is the leak being prevented. Registered as `kvshare_orphan`. Refs #698. --- CMakeLists.txt | 2 ++ tests/kvshare_fd_handoff.py | 19 ++++++++++- tests/kvshare_orphan_test.sh | 66 ++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 tests/kvshare_orphan_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 324beeebb..e4414b6d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -243,6 +243,8 @@ add_test(NAME release_check_row COMMAND ${CMAKE_SOURCE_DIR}/os/mini/release-chec # of the ciphertext, Ed25519 over the manifest) and checks it; the test is a throwaway set in a temp # dir, so no key, no device and no build: the four refusals are a set with no manifest, a file the # manifest does not list, a ciphertext byte flipped after signing, and a manifest signed by another key. +# the handoff driver must not orphan its server on SIGKILL (PR_SET_PDEATHSIG); no GPU, no model +add_test(NAME kvshare_orphan COMMAND ${CMAKE_SOURCE_DIR}/tests/kvshare_orphan_test.sh) add_test(NAME seal_manifest COMMAND ${CMAKE_SOURCE_DIR}/tests/seal_manifest_test.sh) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that diff --git a/tests/kvshare_fd_handoff.py b/tests/kvshare_fd_handoff.py index 0836c9015..93362d8de 100755 --- a/tests/kvshare_fd_handoff.py +++ b/tests/kvshare_fd_handoff.py @@ -47,6 +47,22 @@ CTX = os.environ.get("CTX", "4096") +def die_with_parent(): + """Have the kernel kill the server if this process dies, however it dies. + + The `finally` below reaps the server on a clean exit, a Ctrl-C and a SIGTERM, but not on SIGKILL (an OOM kill, + a CI timeout, `kill -9` by an operator): the child is in its own session, so it would survive and keep its HRX + context -- and on this box an orphaned llama-server held 51 GiB of GTT for 14 h and knocked out another + service. PR_SET_PDEATHSIG closes that case: the kernel signals the child when the parent dies, whatever the + reason. Linux only, which is where the HRX route runs; elsewhere the flag is simply not set. + """ + try: + import ctypes, signal + ctypes.CDLL("libc.so.6", use_errno=True).prctl(1, signal.SIGTERM) # PR_SET_PDEATHSIG + except Exception: + pass + + def free_port(): s = socket.socket() s.bind(("127.0.0.1", 0)) @@ -133,7 +149,8 @@ def say(msg): srv = subprocess.Popen([SRV, "-m", MODEL, "--host", "127.0.0.1", "--port", str(port), "--device", "HRX0", "-ngl", "99", "--jinja", "-c", CTX, "-np", "1", "-fa", "on", "--no-warmup", "--slot-save-path", slotdir], - env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True) + env=env, stdout=lf, stderr=subprocess.STDOUT, start_new_session=True, + preexec_fn=die_with_parent) try: up, secs = wait_health(api, srv, out) say(f"server pid {srv.pid} on :{port}, health up={up} after {secs:.1f}s") diff --git a/tests/kvshare_orphan_test.sh b/tests/kvshare_orphan_test.sh new file mode 100755 index 000000000..647801ea2 --- /dev/null +++ b/tests/kvshare_orphan_test.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The handoff driver must not be able to orphan its server. tests/kvshare_fd_handoff.py runs an HRX llama-server in +# its own session, so the `finally` that reaps it covers a clean exit, Ctrl-C and SIGTERM -- but not SIGKILL (OOM, +# a CI timeout, `kill -9`). An orphaned llama-server held 51 GiB of GTT for 14 h on this box on 2026-11-08 and +# knocked out another service, so the child is spawned with PR_SET_PDEATHSIG. This pins that: SIGKILL the parent +# and the child must go with it. +# +# No GPU and no model: the child is a sleep, spawned through the driver's own `die_with_parent`. +set -uo pipefail +here=$(cd "$(dirname "$0")" && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } + +python3 - "$here" > "$scratch/child.pid" <<'PY' & +import subprocess, sys, time +sys.path.insert(0, sys.argv[1]) +import kvshare_fd_handoff as k +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"], + preexec_fn=k.die_with_parent, start_new_session=True) +print(child.pid, flush=True) +time.sleep(120) +PY +parent=$! +for _ in $(seq 1 50); do [ -s "$scratch/child.pid" ] && break; sleep 0.1; done +child=$(cat "$scratch/child.pid" 2>/dev/null || true) +check "the driver spawned a child in its own session (pid ${child:-none})" '[ -n "$child" ] && kill -0 "$child" 2>/dev/null' + +kill -9 "$parent" 2>/dev/null +wait "$parent" 2>/dev/null +gone=0 +for _ in $(seq 1 40); do kill -0 "$child" 2>/dev/null || { gone=1; break; }; sleep 0.05; done +check "SIGKILL of the driver takes the child with it (PR_SET_PDEATHSIG)" '[ "$gone" = 1 ]' + +# and the negative control: a child spawned *without* it survives, which is what the flag is protecting against +python3 - "$here" > "$scratch/leak.pid" <<'PY' & +import subprocess, sys, time +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) +print(child.pid, flush=True) +time.sleep(120) +PY +parent=$! +for _ in $(seq 1 50); do [ -s "$scratch/leak.pid" ] && break; sleep 0.1; done +leak=$(cat "$scratch/leak.pid" 2>/dev/null || true) +kill -9 "$parent" 2>/dev/null; wait "$parent" 2>/dev/null; sleep 0.3 +check "without the flag the child is orphaned (the case this test exists for)" '[ -n "$leak" ] && kill -0 "$leak" 2>/dev/null' +kill -9 "$leak" 2>/dev/null + +if [ $fail -ne 0 ]; then echo FAIL; exit 1; fi +echo PASS