From 3faf5d9feece6fe13ba08fd79b1d12bf8249ec6d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 18:41:35 -0400 Subject: [PATCH 1/2] tests: laya_models_e2e and laya_route_e2e take the port serve reports; census moves with them Batch 5 of #1485, lock list in the same commit again. - `laya_models_e2e.sh`: the three refusal cases (`--laya-models` without `--laya`, with `-m`, and a table whose class has no row) are `--port 0` with no parse - they never bind; the live serve reads the port back and only then builds `$api`. - `laya_route_e2e.sh`: same split - the "no checkpoint" refusal is `--port 0`, and the live serve reads its port from the ready line before `$api` is built. - Both were building `$api` from a guessed port *before* any serve had started, which is why the parse has to come before it in the live case. - `list(APPEND _onebit_route_tests ...)` loses `laya_route_e2e` and `laya_models_e2e`: with `--port 0` there is no window for the lock to close. The bare-build census floor is unchanged at 10 - these two only exist in model-enabled builds. Census rehearsed against a stand-in `ctest`: 3/3 pass. `bash -n` clean; shellcheck counts unchanged against main (laya_models_e2e 4, laya_route_e2e 7). Left: `npu_repack_path` (needs an NPU device and runs serve inside `timeout`, so the pid to watch is the wrapper's) and `kvshare_fd_handoff.py` (python, spawns serve itself). Both need their own read rather than a substitution, and `hrx_cpu_weight_parity` stays out permanently (llama-server, not `1bit serve`). --- CMakeLists.txt | 6 +++--- tests/laya_models_e2e.sh | 17 +++++++++++------ tests/laya_route_e2e.sh | 13 +++++++++---- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cd8902f2..0d8a5f29f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1455,10 +1455,10 @@ set(_onebit_route_tests smoke_serve print_backend lemonade_hrx template_pin device_route npu_repack_path pm_route bench_selftest hrx_cwsr_route hybrid_route hybrid_kvshare_route) # Tests that only exist for some configurations (a model was passed, a device is built in). -# serve_e2e_ left this list with engine #1485: it runs `serve --port 0` now, so it picks nothing and there is no -# window for the lock to close. hrx_cpu_weight_parity stays - it starts `llama-server` itself, whose port is not ours to +# serve_e2e_, laya_route_e2e and laya_models_e2e left this list with engine #1485: they run `serve --port 0` +# now, so they pick nothing and there is no window for the lock to close. hrx_cpu_weight_parity stays - it starts `llama-server` itself, whose port is not ours to # choose (that is also why it is not in the `1bit serve` migration). -list(APPEND _onebit_route_tests hybrid_e2e laya_route_e2e laya_models_e2e pm_laya_route hrx_cpu_weight_parity) +list(APPEND _onebit_route_tests hybrid_e2e pm_laya_route hrx_cpu_weight_parity) foreach(_onebit_route_test IN LISTS _onebit_route_tests) if(TEST ${_onebit_route_test}) set_tests_properties(${_onebit_route_test} PROPERTIES RESOURCE_LOCK route_ports) diff --git a/tests/laya_models_e2e.sh b/tests/laya_models_e2e.sh index ed548ad3d..562ea80a9 100755 --- a/tests/laya_models_e2e.sh +++ b/tests/laya_models_e2e.sh @@ -27,6 +27,8 @@ set -uo pipefail bin=${1:?usage: laya_models_e2e.sh path/to/1bit path/to/laya-model-dir} laya_model=${2:?usage: laya_models_e2e.sh path/to/1bit path/to/laya-model-dir} here=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=tests/serve_port.sh +. "$here/serve_port.sh" # serve_port_wait: read back the port serve bound instead of guessing one (engine #1485) scratch=$(mktemp -d) touch "$scratch/small.gguf" "$scratch/big.gguf" fake="$here/fake_backend_route.py" @@ -39,21 +41,24 @@ cat >"$scratch/table.json" <&1) +out=$("$bin" serve --laya-models "$scratch/table.json" --port 0 2>&1) check "--laya-models without --laya says so" '[[ "$out" == *"needs --laya"* ]]' -out=$("$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" -m "$scratch/big.gguf" --port "$port" 2>&1) +out=$("$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" -m "$scratch/big.gguf" --port 0 2>&1) check "--laya-models with -m is refused" '[[ "$out" == *"drop -m"* ]]' echo '{"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx"}}}' >"$scratch/bad.json" -out=$("$bin" serve --laya-models "$scratch/bad.json" --laya-model "$laya_model" --port "$port" 2>&1) +out=$("$bin" serve --laya-models "$scratch/bad.json" --laya-model "$laya_model" --port 0 2>&1) check "a table with a class and no row is refused by name" '[[ "$out" == *"\"b\" has no row"* ]]' -"$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" --port "$port" >"$scratch/serve.log" 2>&1 & +"$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" --port 0 >"$scratch/serve.log" 2>&1 & pid=$! +port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" cleanup() { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; } trap cleanup EXIT for _ in $(seq 1 1200); do [ "$(curl -s -o /dev/null -w '%{http_code}' "$api/health")" = 200 ] && break; sleep 0.1; done diff --git a/tests/laya_route_e2e.sh b/tests/laya_route_e2e.sh index 1d178b1a3..0c652efea 100755 --- a/tests/laya_route_e2e.sh +++ b/tests/laya_route_e2e.sh @@ -30,6 +30,8 @@ set -uo pipefail bin=${1:?usage: laya_route_e2e.sh path/to/1bit path/to/laya-model-dir} laya_model=${2:?usage: laya_route_e2e.sh path/to/1bit path/to/laya-model-dir} here=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=tests/serve_port.sh +. "$here/serve_port.sh" # serve_port_wait: read back the port serve bound instead of guessing one (engine #1485) policy="$here/route-policy-e2e.json" # The devices serve offers Laya for a .gguf in this build (gguf_devices() in app/serve.cpp): HRX # and the CPU in a build with HRX; the CPU only in one without (CI). @@ -39,8 +41,9 @@ if grep -q "^ONEBIT_HRX:BOOL=ON" "$cache" 2>/dev/null; then devices=hrx,cpu; fi echo "candidates: $devices" scratch=$(mktemp -d) touch "$scratch/tiny.gguf" -port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') -api="http://127.0.0.1:$port" +# --port 0: the refusal case below never binds, and the live serve reads its port back from serve's own ready line +# (engine #1444/#1485); nothing picks a port and releases it before serve binds. +api= fail=0 check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } @@ -48,14 +51,16 @@ npu_only=$("$bin" route --laya-model "$laya_model" --devices npu --state "anythi check "--devices npu routes to npu ($npu_only)" '[ "$npu_only" = npu ]' missing=$(HOME="$scratch" XDG_DATA_HOME= ONEBIT_LAYA_MODEL= "$bin" serve -m "$scratch/tiny.gguf" --device auto --laya \ - --port "$port" --llama-server "$here/fake_backend_route.py" 2>&1) + --port 0 --llama-server "$here/fake_backend_route.py" 2>&1) check "--laya without a checkpoint says how to install one" '[[ "$missing" == *fetch-laya.sh* ]]' ONEBIT_LAYA_MODEL="$laya_model" "$bin" serve -m "$scratch/tiny.gguf" --device auto --laya --route-policy "$policy" \ - --port "$port" --alias route-test \ + --port 0 --alias route-test \ --llama-server "$here/fake_backend_route.py" \ >"$scratch/serve.log" 2>&1 & pid=$! +port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" cleanup() { kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; } trap cleanup EXIT for _ in $(seq 1 1200); do curl -s -o /dev/null "$api/health" && break; sleep 0.1; done From 355a3a2f4bec6c2c0667864aa112ebc9103ae027 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 8 Nov 2026 19:04:41 -0400 Subject: [PATCH 2/2] tests: install the cleanup trap before the port wait, so a failed wait cannot orphan a serve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding on #1561, and it is the fleet's own orphan mode: `serve_port_wait` returns 1 while the process is still **alive** (it only breaks early on a dead pid), so the `|| { …; exit 1; }` path is precisely the case where a live serve would be left behind - and the trap that would have killed it was installed two lines below the exit. On this box a serve left behind holds the HRX device and its GTT, which is the same failure as the 14 h llama-server. Both scripts now define `cleanup` and `trap cleanup EXIT` immediately after `pid=$!`, before the wait. That also cleans `$scratch` on the failure path, which leaked with the process. Shown, not asserted: a stubbed script with the new ordering leaves no child after the failure path, the same script with the old ordering leaves one running. `bash -n` clean; shellcheck counts unchanged against main. --- tests/laya_models_e2e.sh | 7 +++++-- tests/laya_route_e2e.sh | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/laya_models_e2e.sh b/tests/laya_models_e2e.sh index 562ea80a9..c3851978e 100755 --- a/tests/laya_models_e2e.sh +++ b/tests/laya_models_e2e.sh @@ -57,10 +57,13 @@ check "a table with a class and no row is refused by name" '[[ "$out" == *"\"b\" "$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" --port 0 >"$scratch/serve.log" 2>&1 & pid=$! -port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } -api="http://127.0.0.1:$port" +# The trap goes in BEFORE the wait below: serve_port_wait returns 1 while the process is still alive (it only breaks early +# on a dead pid), so the failure path is exactly the case where a live serve would be left behind - and a serve left behind +# on this box holds the HRX device and its GTT. Same rule the fleet learned from the 14 h llama-server (engine #1485 review). cleanup() { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; } trap cleanup EXIT +port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" for _ in $(seq 1 1200); do [ "$(curl -s -o /dev/null -w '%{http_code}' "$api/health")" = 200 ] && break; sleep 0.1; done check "ready" '[ "$(curl -s -o /dev/null -w "%{http_code}" "$api/health")" = 200 ]' diff --git a/tests/laya_route_e2e.sh b/tests/laya_route_e2e.sh index 0c652efea..7b50d121e 100755 --- a/tests/laya_route_e2e.sh +++ b/tests/laya_route_e2e.sh @@ -59,10 +59,12 @@ ONEBIT_LAYA_MODEL="$laya_model" "$bin" serve -m "$scratch/tiny.gguf" --device au --llama-server "$here/fake_backend_route.py" \ >"$scratch/serve.log" 2>&1 & pid=$! -port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } -api="http://127.0.0.1:$port" +# trap first: serve_port_wait's failure case is a serve that is still alive, so exiting before this line would leave it +# running with the device open (engine #1485 review; the 14 h llama-server is the same failure mode) cleanup() { kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; } trap cleanup EXIT +port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" for _ in $(seq 1 1200); do curl -s -o /dev/null "$api/health" && break; sleep 0.1; done states=(