Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1455,10 +1455,10 @@ set(_onebit_route_tests
smoke_serve print_backend lemonade_hrx template_pin device_route npu_repack_path pm_route bench_selftest
hrx_cwsr_route hybrid_route hybrid_kvshare_route)
# Tests that only exist for some configurations (a model was passed, a device is built in).
# serve_e2e_<dev> left this list with engine #1485: it runs `serve --port 0` now, so it picks nothing and there is no
# window for the lock to close. hrx_cpu_weight_parity stays - it starts `llama-server` itself, whose port is not ours to
# serve_e2e_<dev>, laya_route_e2e and laya_models_e2e left this list with engine #1485: they run `serve --port 0`
# now, so they pick nothing and there is no window for the lock to close. hrx_cpu_weight_parity stays - it starts `llama-server` itself, whose port is not ours to
# choose (that is also why it is not in the `1bit serve` migration).
list(APPEND _onebit_route_tests hybrid_e2e laya_route_e2e laya_models_e2e pm_laya_route hrx_cpu_weight_parity)
list(APPEND _onebit_route_tests hybrid_e2e pm_laya_route hrx_cpu_weight_parity)
foreach(_onebit_route_test IN LISTS _onebit_route_tests)
if(TEST ${_onebit_route_test})
set_tests_properties(${_onebit_route_test} PROPERTIES RESOURCE_LOCK route_ports)
Expand Down
20 changes: 14 additions & 6 deletions tests/laya_models_e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ set -uo pipefail
bin=${1:?usage: laya_models_e2e.sh path/to/1bit path/to/laya-model-dir}
laya_model=${2:?usage: laya_models_e2e.sh path/to/1bit path/to/laya-model-dir}
here=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/serve_port.sh
. "$here/serve_port.sh" # serve_port_wait: read back the port serve bound instead of guessing one (engine #1485)
scratch=$(mktemp -d)
touch "$scratch/small.gguf" "$scratch/big.gguf"
fake="$here/fake_backend_route.py"
Expand All @@ -39,23 +41,29 @@ cat >"$scratch/table.json" <<EOF
"classes": {"reflex": {"model": "small.gguf", "device": "cpu", "alias": "small-npu", "args": ["--llama-server", "$fake"]},
"plan": {"model": "$scratch/big.gguf", "device": "cpu", "alias": "big-hrx", "args": ["--llama-server", "$fake"]}}}
EOF
port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')
api="http://127.0.0.1:$port"
# --port 0 everywhere below: the three refusal cases never bind, and the live one reads the port back from serve's own
# ready line, so nothing picks a port and releases it before serve binds (engine #1444/#1485).
api=
fail=0
check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; }

out=$("$bin" serve --laya-models "$scratch/table.json" --port "$port" 2>&1)
out=$("$bin" serve --laya-models "$scratch/table.json" --port 0 2>&1)
check "--laya-models without --laya says so" '[[ "$out" == *"needs --laya"* ]]'
out=$("$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" -m "$scratch/big.gguf" --port "$port" 2>&1)
out=$("$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" -m "$scratch/big.gguf" --port 0 2>&1)
check "--laya-models with -m is refused" '[[ "$out" == *"drop -m"* ]]'
echo '{"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx"}}}' >"$scratch/bad.json"
out=$("$bin" serve --laya-models "$scratch/bad.json" --laya-model "$laya_model" --port "$port" 2>&1)
out=$("$bin" serve --laya-models "$scratch/bad.json" --laya-model "$laya_model" --port 0 2>&1)
check "a table with a class and no row is refused by name" '[[ "$out" == *"\"b\" has no row"* ]]'

"$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" --port "$port" >"$scratch/serve.log" 2>&1 &
"$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" --port 0 >"$scratch/serve.log" 2>&1 &
pid=$!
# The trap goes in BEFORE the wait below: serve_port_wait returns 1 while the process is still alive (it only breaks early
# on a dead pid), so the failure path is exactly the case where a live serve would be left behind - and a serve left behind
# on this box holds the HRX device and its GTT. Same rule the fleet learned from the 14 h llama-server (engine #1485 review).
cleanup() { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; }
trap cleanup EXIT
port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; }
api="http://127.0.0.1:$port"
for _ in $(seq 1 1200); do [ "$(curl -s -o /dev/null -w '%{http_code}' "$api/health")" = 200 ] && break; sleep 0.1; done
check "ready" '[ "$(curl -s -o /dev/null -w "%{http_code}" "$api/health")" = 200 ]'

Expand Down
15 changes: 11 additions & 4 deletions tests/laya_route_e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ set -uo pipefail
bin=${1:?usage: laya_route_e2e.sh path/to/1bit path/to/laya-model-dir}
laya_model=${2:?usage: laya_route_e2e.sh path/to/1bit path/to/laya-model-dir}
here=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/serve_port.sh
. "$here/serve_port.sh" # serve_port_wait: read back the port serve bound instead of guessing one (engine #1485)
policy="$here/route-policy-e2e.json"
# The devices serve offers Laya for a .gguf in this build (gguf_devices() in app/serve.cpp): HRX
# and the CPU in a build with HRX; the CPU only in one without (CI).
Expand All @@ -39,25 +41,30 @@ if grep -q "^ONEBIT_HRX:BOOL=ON" "$cache" 2>/dev/null; then devices=hrx,cpu; fi
echo "candidates: $devices"
scratch=$(mktemp -d)
touch "$scratch/tiny.gguf"
port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')
api="http://127.0.0.1:$port"
# --port 0: the refusal case below never binds, and the live serve reads its port back from serve's own ready line
# (engine #1444/#1485); nothing picks a port and releases it before serve binds.
api=
fail=0
check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; }

npu_only=$("$bin" route --laya-model "$laya_model" --devices npu --state "anything" 2>/dev/null)
check "--devices npu routes to npu ($npu_only)" '[ "$npu_only" = npu ]'

missing=$(HOME="$scratch" XDG_DATA_HOME= ONEBIT_LAYA_MODEL= "$bin" serve -m "$scratch/tiny.gguf" --device auto --laya \
--port "$port" --llama-server "$here/fake_backend_route.py" 2>&1)
--port 0 --llama-server "$here/fake_backend_route.py" 2>&1)
check "--laya without a checkpoint says how to install one" '[[ "$missing" == *fetch-laya.sh* ]]'

ONEBIT_LAYA_MODEL="$laya_model" "$bin" serve -m "$scratch/tiny.gguf" --device auto --laya --route-policy "$policy" \
--port "$port" --alias route-test \
--port 0 --alias route-test \
--llama-server "$here/fake_backend_route.py" \
>"$scratch/serve.log" 2>&1 &
pid=$!
# trap first: serve_port_wait's failure case is a serve that is still alive, so exiting before this line would leave it
# running with the device open (engine #1485 review; the 14 h llama-server is the same failure mode)
cleanup() { kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; }
trap cleanup EXIT
port=$(serve_port_wait "$scratch/serve.log" "$pid" 1200) || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; }
api="http://127.0.0.1:$port"
for _ in $(seq 1 1200); do curl -s -o /dev/null "$api/health" && break; sleep 0.1; done

states=(
Expand Down
Loading