From bcd7c2453439b945b6fe59039555b9a9e58ad66a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 10:18:51 -0400 Subject: [PATCH 1/3] tests: test-mtp-one-request, one llama-server request with MTP drafting on the CPU (engine#2433) Runs llama-server -ngl 0 --spec-type draft-mtp on ONEBIT_MTP_TEST_MODEL (a GGUF with its own MTP head) and sends one /completion request. Passes only on HTTP 200, non-empty content, a live server afterwards and a 'draft acceptance' line, so the MTP drafter really ran. Skips (77) without a model. CPU only, so it needs no GPU. On the fork at ea2d2eb07 the server segfaults in the drafter (engine#2433) and the arm fails; with the drafter's budget.begin restored it passes. Co-Authored-By: Claude Opus 5.5 --- tests/CMakeLists.txt | 12 ++++++++++ tests/test-mtp-one-request.sh | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100755 tests/test-mtp-one-request.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7ba2d6051d2d..3f2d6aaa46ae 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -153,6 +153,18 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen35 LABEL behavior llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-refact.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-starcoder LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-starcoder.gguf) +# engine#2433 regression arm: one llama-server request with MTP drafting, CPU only; needs ONEBIT_MTP_TEST_MODEL +# (a GGUF with its own MTP head), else it reports SKIP (77) +if (NOT WIN32 AND LLAMA_BUILD_SERVER) + llama_test_cmd( + ${CMAKE_CURRENT_SOURCE_DIR}/test-mtp-one-request.sh + NAME test-mtp-one-request + LABEL behavior + ARGS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + ) + set_tests_properties(test-mtp-one-request PROPERTIES SKIP_RETURN_CODE 77) +endif() + if (NOT WIN32) llama_test_cmd( ${CMAKE_CURRENT_SOURCE_DIR}/test-tokenizers-repo.sh diff --git a/tests/test-mtp-one-request.sh b/tests/test-mtp-one-request.sh new file mode 100755 index 000000000000..519f1bd5c14f --- /dev/null +++ b/tests/test-mtp-one-request.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# engine#2433 regression arm: one request through llama-server with MTP drafting (--spec-type draft-mtp). +# CPU only (-ngl 0), so it needs no GPU. Needs a GGUF that carries its own MTP head: +# ONEBIT_MTP_TEST_MODEL=/path/to/model.gguf (e.g. Qwen3.8-27B UD-Q4_K_XL); unset or missing -> skip (77). +# Pass: HTTP 200 from /completion with non-empty content, the server still alive afterwards, and the server log +# reporting a draft acceptance line (so the MTP drafter really ran). Fail: anything else, including a crash. +set -u +BIN_DIR=${1:?usage: test-mtp-one-request.sh } +M=${ONEBIT_MTP_TEST_MODEL:-} +if [ -z "$M" ] || [ ! -f "$M" ]; then + echo "test-mtp-one-request: ONEBIT_MTP_TEST_MODEL unset or missing, SKIP" + exit 77 +fi +P=${ONEBIT_MTP_TEST_PORT:-$((20000 + RANDOM % 20000))} +LOG=$(mktemp -p "${TMPDIR:-$BIN_DIR}" mtp-one-request.XXXXXX.log) +"$BIN_DIR/llama-server" -m "$M" -ngl 0 -c 2048 -np 1 --host 127.0.0.1 --port "$P" \ + --spec-type draft-mtp --spec-draft-n-max 3 > "$LOG" 2>&1 & +SRV=$! +cleanup() { kill "$SRV" 2>/dev/null; wait "$SRV" 2>/dev/null; } +trap cleanup EXIT +code=000 +for _ in $(seq 1 200); do + code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$P/health" || true) + [ "$code" = 200 ] && break + kill -0 "$SRV" 2>/dev/null || break + sleep 3 +done +if [ "$code" != 200 ]; then echo "test-mtp-one-request: server not healthy (last $code), FAIL"; tail -20 "$LOG"; exit 1; fi +out=$(curl -s -m 600 -w '\n%{http_code}' "http://127.0.0.1:$P/completion" \ + -d '{"prompt":"The capital of France is","n_predict":16,"temperature":0}') +http=${out##*$'\n'}; body=${out%$'\n'*} +sleep 1 +alive=yes; kill -0 "$SRV" 2>/dev/null || alive=no +content=$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("content",""))' 2>/dev/null || true) +acc=$(grep -m1 "draft acceptance" "$LOG" || true) +echo "test-mtp-one-request: http=$http alive=$alive content=$(printf '%q' "${content:0:60}")" +echo " ${acc:-no draft acceptance line}" +if [ "$http" = 200 ] && [ -n "$content" ] && [ "$alive" = yes ] && [ -n "$acc" ]; then + echo "test-mtp-one-request: OK"; rm -f "$LOG"; exit 0 +fi +echo "test-mtp-one-request: FAIL (log $LOG)"; grep -E "SIGSEGV|signal|error" "$LOG" | tail -5 +exit 1 From beb482de3282d0c5f9c5386201825b930687cfb2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 11:47:08 -0400 Subject: [PATCH 2/3] tests: test-mtp-one-request gives each failure its own exit code, retries a taken port, has a 900 s timeout (review on #379) Exit 1 never healthy, 2 server died (the #2433 SIGSEGV), 3 request failed or timed out, 4 empty content, 5 no draft acceptance line. The port comes from the OS and a bind failure retries on a fresh one. The ctest row has TIMEOUT 900, and both files say that CI leaves ONEBIT_MTP_TEST_MODEL unset, so the row is SKIP there. Co-Authored-By: Claude Opus 5.5 --- tests/CMakeLists.txt | 3 +- tests/test-mtp-one-request.sh | 53 ++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3f2d6aaa46ae..40589360fd39 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -162,7 +162,8 @@ if (NOT WIN32 AND LLAMA_BUILD_SERVER) LABEL behavior ARGS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} ) - set_tests_properties(test-mtp-one-request PROPERTIES SKIP_RETURN_CODE 77) + # CI leaves ONEBIT_MTP_TEST_MODEL unset, so the row is SKIP there; the timeout bounds a slow CPU load + request + set_tests_properties(test-mtp-one-request PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 900) endif() if (NOT WIN32) diff --git a/tests/test-mtp-one-request.sh b/tests/test-mtp-one-request.sh index 519f1bd5c14f..80108580a84d 100755 --- a/tests/test-mtp-one-request.sh +++ b/tests/test-mtp-one-request.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash # engine#2433 regression arm: one request through llama-server with MTP drafting (--spec-type draft-mtp). # CPU only (-ngl 0), so it needs no GPU. Needs a GGUF that carries its own MTP head: -# ONEBIT_MTP_TEST_MODEL=/path/to/model.gguf (e.g. Qwen3.8-27B UD-Q4_K_XL); unset or missing -> skip (77). +# ONEBIT_MTP_TEST_MODEL=/path/to/model.gguf (e.g. Qwen3.8-27B UD-Q4_K_XL); unset or missing -> SKIP (77). +# CI does not set ONEBIT_MTP_TEST_MODEL, so this row reports SKIP there; it runs on a box that has the GGUF. # Pass: HTTP 200 from /completion with non-empty content, the server still alive afterwards, and the server log -# reporting a draft acceptance line (so the MTP drafter really ran). Fail: anything else, including a crash. +# reporting a draft acceptance line (so the MTP drafter really ran). Distinct exit codes per failure: +# 1 server never became healthy 2 the server died (e.g. the #2433 SIGSEGV) 3 the request timed out or HTTP != 200 +# 4 empty content 5 no draft acceptance line (the drafter did not run) set -u BIN_DIR=${1:?usage: test-mtp-one-request.sh } M=${ONEBIT_MTP_TEST_MODEL:-} @@ -11,21 +14,37 @@ if [ -z "$M" ] || [ ! -f "$M" ]; then echo "test-mtp-one-request: ONEBIT_MTP_TEST_MODEL unset or missing, SKIP" exit 77 fi -P=${ONEBIT_MTP_TEST_PORT:-$((20000 + RANDOM % 20000))} LOG=$(mktemp -p "${TMPDIR:-$BIN_DIR}" mtp-one-request.XXXXXX.log) -"$BIN_DIR/llama-server" -m "$M" -ngl 0 -c 2048 -np 1 --host 127.0.0.1 --port "$P" \ - --spec-type draft-mtp --spec-draft-n-max 3 > "$LOG" 2>&1 & -SRV=$! -cleanup() { kill "$SRV" 2>/dev/null; wait "$SRV" 2>/dev/null; } +SRV= +cleanup() { [ -n "$SRV" ] && { kill "$SRV" 2>/dev/null; wait "$SRV" 2>/dev/null; }; } trap cleanup EXIT +free_port() { python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'; } + +# A port the OS reports free can still be taken before llama-server binds it, so a bind failure retries on a new one. code=000 -for _ in $(seq 1 200); do - code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$P/health" || true) +for attempt in 1 2 3; do + P=${ONEBIT_MTP_TEST_PORT:-$(free_port)} + "$BIN_DIR/llama-server" -m "$M" -ngl 0 -c 2048 -np 1 --host 127.0.0.1 --port "$P" \ + --spec-type draft-mtp --spec-draft-n-max 3 > "$LOG" 2>&1 & + SRV=$! + for _ in $(seq 1 200); do + code=$(curl -s -m 2 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$P/health" || true) + [ "$code" = 200 ] && break + kill -0 "$SRV" 2>/dev/null || break + sleep 3 + done [ "$code" = 200 ] && break - kill -0 "$SRV" 2>/dev/null || break - sleep 3 + if grep -qiE "address already in use|couldn't bind|failed to bind" "$LOG" && [ -z "${ONEBIT_MTP_TEST_PORT:-}" ]; then + echo "test-mtp-one-request: port $P taken, retrying ($attempt)"; cleanup; SRV=; continue + fi + break done -if [ "$code" != 200 ]; then echo "test-mtp-one-request: server not healthy (last $code), FAIL"; tail -20 "$LOG"; exit 1; fi +if [ "$code" != 200 ]; then + if [ -n "$SRV" ] && ! kill -0 "$SRV" 2>/dev/null; then + echo "test-mtp-one-request: server DIED before it was healthy, FAIL"; tail -20 "$LOG"; exit 2 + fi + echo "test-mtp-one-request: server never healthy (last $code), FAIL"; tail -20 "$LOG"; exit 1 +fi out=$(curl -s -m 600 -w '\n%{http_code}' "http://127.0.0.1:$P/completion" \ -d '{"prompt":"The capital of France is","n_predict":16,"temperature":0}') http=${out##*$'\n'}; body=${out%$'\n'*} @@ -35,8 +54,10 @@ content=$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys acc=$(grep -m1 "draft acceptance" "$LOG" || true) echo "test-mtp-one-request: http=$http alive=$alive content=$(printf '%q' "${content:0:60}")" echo " ${acc:-no draft acceptance line}" -if [ "$http" = 200 ] && [ -n "$content" ] && [ "$alive" = yes ] && [ -n "$acc" ]; then - echo "test-mtp-one-request: OK"; rm -f "$LOG"; exit 0 +if [ "$alive" != yes ]; then + echo "test-mtp-one-request: the server DIED during the request, FAIL (log $LOG)"; grep -E "SIGSEGV|signal|error" "$LOG" | tail -5; exit 2 fi -echo "test-mtp-one-request: FAIL (log $LOG)"; grep -E "SIGSEGV|signal|error" "$LOG" | tail -5 -exit 1 +if [ "$http" != 200 ]; then echo "test-mtp-one-request: request failed or timed out (http=$http), FAIL (log $LOG)"; exit 3; fi +if [ -z "$content" ]; then echo "test-mtp-one-request: empty content, FAIL (log $LOG)"; exit 4; fi +if [ -z "$acc" ]; then echo "test-mtp-one-request: no draft acceptance line, the MTP drafter did not run, FAIL (log $LOG)"; exit 5; fi +echo "test-mtp-one-request: OK"; rm -f "$LOG"; exit 0 From a994c9270ec88702c54200be86714f7d7ac83d3e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 12:06:50 -0400 Subject: [PATCH 3/3] tests: test-mtp-one-request needs at least one accepted draft, and its ctest TIMEOUT covers its own budget (review on #379) A 'draft acceptance' line reading 0 accepted now fails (exit 5): the row exists to catch a drafter that does not work. TIMEOUT 1800 exceeds the script's worst case (health 200 x 5 s + the 600 s request, ~1600 s), so a slow load reaches its own exit code instead of a ctest Timeout; the invariant is written next to the number. Co-Authored-By: Claude Opus 5.5 --- tests/CMakeLists.txt | 5 +++-- tests/test-mtp-one-request.sh | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 40589360fd39..8d7ac0dabfff 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -162,8 +162,9 @@ if (NOT WIN32 AND LLAMA_BUILD_SERVER) LABEL behavior ARGS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} ) - # CI leaves ONEBIT_MTP_TEST_MODEL unset, so the row is SKIP there; the timeout bounds a slow CPU load + request - set_tests_properties(test-mtp-one-request PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 900) + # CI leaves ONEBIT_MTP_TEST_MODEL unset, so the row is SKIP there. TIMEOUT must exceed the script's own worst + # case (health 200 x 5 s + the 600 s request = ~1600 s), or a slow load is cut off before its exit code prints + set_tests_properties(test-mtp-one-request PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 1800) endif() if (NOT WIN32) diff --git a/tests/test-mtp-one-request.sh b/tests/test-mtp-one-request.sh index 80108580a84d..e2e5f21c74a8 100755 --- a/tests/test-mtp-one-request.sh +++ b/tests/test-mtp-one-request.sh @@ -6,7 +6,9 @@ # Pass: HTTP 200 from /completion with non-empty content, the server still alive afterwards, and the server log # reporting a draft acceptance line (so the MTP drafter really ran). Distinct exit codes per failure: # 1 server never became healthy 2 the server died (e.g. the #2433 SIGSEGV) 3 the request timed out or HTTP != 200 -# 4 empty content 5 no draft acceptance line (the drafter did not run) +# 4 empty content 5 no draft acceptance line, or 0 drafts accepted (the drafter did not work) +# Worst case of this script: health 200 x (curl -m 2 + sleep 3) = 1000 s + the request (-m 600) = ~1600 s; the ctest +# TIMEOUT must stay above that or these exit codes never get to run. set -u BIN_DIR=${1:?usage: test-mtp-one-request.sh } M=${ONEBIT_MTP_TEST_MODEL:-} @@ -52,6 +54,8 @@ sleep 1 alive=yes; kill -0 "$SRV" 2>/dev/null || alive=no content=$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("content",""))' 2>/dev/null || true) acc=$(grep -m1 "draft acceptance" "$LOG" || true) +# "draft acceptance = 0.64286 ( 9 accepted / 14 generated)": the accepted count is what says the drafter worked +acc_n=$(printf '%s' "$acc" | sed -nE 's/.*\( *([0-9]+) accepted.*/\1/p') echo "test-mtp-one-request: http=$http alive=$alive content=$(printf '%q' "${content:0:60}")" echo " ${acc:-no draft acceptance line}" if [ "$alive" != yes ]; then @@ -60,4 +64,5 @@ fi if [ "$http" != 200 ]; then echo "test-mtp-one-request: request failed or timed out (http=$http), FAIL (log $LOG)"; exit 3; fi if [ -z "$content" ]; then echo "test-mtp-one-request: empty content, FAIL (log $LOG)"; exit 4; fi if [ -z "$acc" ]; then echo "test-mtp-one-request: no draft acceptance line, the MTP drafter did not run, FAIL (log $LOG)"; exit 5; fi +if [ -z "$acc_n" ] || [ "$acc_n" -lt 1 ]; then echo "test-mtp-one-request: 0 drafts accepted (${acc_n:-unparsed}), the MTP drafter did not work, FAIL (log $LOG)"; exit 5; fi echo "test-mtp-one-request: OK"; rm -f "$LOG"; exit 0