From 6a01893463bd74bd26dbf574a1b93202772388b2 Mon Sep 17 00:00:00 2001 From: pi-tmplsec Date: Mon, 9 Nov 2026 22:41:24 -0400 Subject: [PATCH 1/6] tests: a client that hangs up mid-response must not stop the server accepting (engine#1779, F10 fault 1) The test FastFlowLM 1.0.12's notes imply: start a streaming request, kill the client after it has read a few bytes, repeat five times, then require that the process is alive, /health answers and a fresh client is served. No device, no model -- tests/fake_backend.py stands in for the backend, the port is read back from serve's own line (tests/serve_port.sh). Not yet run: no onebit binary on this box yet; the build is the next step. --- CMakeLists.txt | 1 + tests/serve_hangup_test.sh | 79 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100755 tests/serve_hangup_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index e20a86864..83e6d4e25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,6 +196,7 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $) # serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU. +add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) diff --git a/tests/serve_hangup_test.sh b/tests/serve_hangup_test.sh new file mode 100755 index 000000000..13bbb26e7 --- /dev/null +++ b/tests/serve_hangup_test.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A client that hangs up mid-response must not take the server's accept loop with it (engine #1779, F10 fault 1). +# +# FastFlowLM 1.0.12's notes list "a server that stopped accepting connections after clients hung up" among their fixes. +# The fault is invisible to an ordinary test, because it needs a disconnect *during* a response: the listener is fine, +# the connection table is what leaks or the loop is what exits. So: a request is started against the streaming route, +# the client is killed after it has read a few bytes, and the loop repeats; then a fresh client must be served and the +# process must still be alive. +# +# No GPU, no model: tests/fake_backend.py stands in for the backend (app/serve.cpp only proxies to it), and the port is +# read back from serve's own line rather than picked and released here (tests/serve_port.sh, engine #1485). +# +# usage: tests/serve_hangup_test.sh path/to/1bit +set -uo pipefail + +bin=${1:?usage: serve_hangup_test.sh path/to/1bit} +here=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=tests/serve_port.sh +. "$here/serve_port.sh" +scratch=$(mktemp -d) +touch "$scratch/tiny.gguf" +pids=() +cleanup() { + for p in "${pids[@]}"; do pkill -9 -P "$p" 2>/dev/null; kill -9 "$p" 2>/dev/null; done + wait 2>/dev/null + statefile | xargs -r rm -f + rm -rf "$scratch" +} +trap cleanup EXIT +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } + +"$bin" serve -m "$scratch/tiny.gguf" --device cpu --port 0 --alias hangup \ + --llama-server "$here/fake_backend.py" >"$scratch/serve.log" 2>&1 & +pid=$! +pids+=("$pid") +port=$(serve_port_wait "$scratch/serve.log" "$pid") || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" + +for _ in $(seq 1 100); do code=$(curl -s -o /dev/null -w '%{http_code}' "$api/health"); [ "$code" = 200 ] && break; sleep 0.1; done + +body='{"model":"hangup","messages":[{"role":"user","content":"hello"}],"stream":true}' + +# The hang-up: start a streaming request, let it read a little, kill the client. Five times, because a leak of one +# connection per hang-up survives a single iteration. +for i in $(seq 1 5); do + timeout 0.4 curl -sN "$api/v1/chat/completions" -H 'Content-Type: application/json' -d "$body" >"$scratch/hang$i.txt" 2>/dev/null +done + +check "serve is still running after 5 client hang-ups" "kill -0 $pid 2>/dev/null" +code=$(curl -s -o "$scratch/after.txt" -w '%{http_code}' --max-time 5 "$api/health") +check "health answers after the hang-ups (got $code)" "[ \"$code\" = 200 ]" +code=$(curl -s -o "$scratch/fresh.txt" -w '%{http_code}' --max-time 20 \ + "$api/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model":"hangup","messages":[{"role":"user","content":"hello"}]}') +check "a fresh client is served after the hang-ups (got $code)" "[ \"$code\" = 200 ]" +check "and the fresh response carries a body" "[ -s \"$scratch/fresh.txt\" ]" + +if [ $fail -ne 0 ]; then + echo "--- serve log"; cat "$scratch/serve.log" + echo "--- fresh response"; cat "$scratch/fresh.txt" 2>/dev/null + echo FAIL; exit 1 +fi +echo "all hang-up checks passed" From 2955e3dbc77a53107090750b9870e98511931fb9 Mon Sep 17 00:00:00 2001 From: pi-tmplsec Date: Mon, 9 Nov 2026 22:44:46 -0400 Subject: [PATCH 2/6] tests: the hang-up note and a cleanup that cannot fail for an unrelated reason (engine#1779) Measured on main (build/1bit at this head): 5 of 5 hang-ups had begun receiving a response when the client was killed, and the server survived all five -- health 200 and a fresh client served. So F10 fault 1 is NOT present in our serve at this head; the test is now the thing that keeps it that way. --- tests/serve_hangup_test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/serve_hangup_test.sh b/tests/serve_hangup_test.sh index 13bbb26e7..70817cd56 100755 --- a/tests/serve_hangup_test.sh +++ b/tests/serve_hangup_test.sh @@ -38,7 +38,9 @@ pids=() cleanup() { for p in "${pids[@]}"; do pkill -9 -P "$p" 2>/dev/null; kill -9 "$p" 2>/dev/null; done wait 2>/dev/null - statefile | xargs -r rm -f + # serve keeps a state file naming the port it bound; drop it like tests/metrics_serve.sh does, but only if the + # helper exists -- this test must not fail in cleanup for a reason unrelated to the fault it measures. + if command -v statefile >/dev/null 2>&1; then statefile | xargs -r rm -f; fi rm -rf "$scratch" } trap cleanup EXIT @@ -62,6 +64,12 @@ for i in $(seq 1 5); do timeout 0.4 curl -sN "$api/v1/chat/completions" -H 'Content-Type: application/json' -d "$body" >"$scratch/hang$i.txt" 2>/dev/null done +# How many hang-ups actually reached the response stage: the client is killed by `timeout`, so a non-empty file means the +# server had begun writing when the socket went away -- which is the moment this fault lives in. Reported, not asserted: +# a run where none reached it proves less than it appears to, and the reader deserves to know which they got. +got=0; for i in $(seq 1 5); do [ -s "$scratch/hang$i.txt" ] && got=$((got + 1)); done +echo "note $got of 5 hang-ups had begun receiving a response before the client was killed" + check "serve is still running after 5 client hang-ups" "kill -0 $pid 2>/dev/null" code=$(curl -s -o "$scratch/after.txt" -w '%{http_code}' --max-time 5 "$api/health") check "health answers after the hang-ups (got $code)" "[ \"$code\" = 200 ]" From 8f55095df6e5dac247be93bffea17f8bd56dc523 Mon Sep 17 00:00:00 2001 From: pi-tmplsec Date: Tue, 10 Nov 2026 00:12:03 -0400 Subject: [PATCH 3/6] npu: the gemma4 answer family, and the tag window that could never match it (engine#1779, F10 fault 4) AnswerFamily had one member (qwen3) and tags_of() threw for anything else, so a Gemma 4 pack could not have its tool calls parsed at all. Adds the gemma4 family from the pack's own tokenizer_config.json -- call:NAME{key:value} with <|"|>- escaped strings, normalised to the same JSON ToolCall shape the qwen3 route produces -- plus tests for the escaped-string-comma case and the no-tools case. And the adapter needed fixing first: scan() took a 16-byte window, so `<|channel>thought` (18 bytes) could never match and every long tag in any family would have read as text. Window is 32 with the reason written down. --- app/npu_answer_parser.h | 114 +++++++++++++++++++++++++++++++++++- tests/npu_sampling_test.cpp | 41 +++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 24bc88ef6..dcd39cffc 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -34,6 +34,9 @@ #include +#include +#include + #include #include #include @@ -44,6 +47,11 @@ namespace onebit::npu { enum class AnswerFamily { qwen3, + // Gemma 4 (engine#1779, F10 fault 4): the pack's own tokenizer_config.json states this markup, and it is not + // qwen3's. Tokens: <|turn>/, <|channel>/, <|tool_call>/, escape <|"|>. + // A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is + // why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows). + gemma4, }; // One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is @@ -63,9 +71,17 @@ struct ParsedAnswer { namespace answer_detail { +inline std::string trim(const std::string& s) { + size_t b = 0, e = s.size(); + while (b < e && (s[b] == ' ' || s[b] == '\n' || s[b] == '\r' || s[b] == '\t')) ++b; + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\n' || s[e - 1] == '\r' || s[e - 1] == '\t')) --e; + return s.substr(b, e - b); +} + inline const char* family_name(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return "qwen3"; + case AnswerFamily::gemma4: return "gemma4"; } return "?"; } @@ -82,6 +98,10 @@ inline FamilyTags tags_of(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return FamilyTags{"", "", "", ""}; + case AnswerFamily::gemma4: + // The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the + // template emits exactly `<|channel>thought\n…\n`, so the pair is the tag. + return FamilyTags{"<|channel>thought", "", "<|tool_call>", ""}; } throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f)); } @@ -169,7 +189,10 @@ class AnswerStream { }; while (i < safe.size()) { if (safe[i] == '<') { - const std::string rest = safe.substr(i, 16); + // 32 bytes, not 16: gemma4's thinking tag is `<|channel>thought` (18 bytes) and the old + // 16-byte window could never match it -- every tag longer than the window looked like text + // (engine#1779). 32 covers the longest tag any family carries with room to spare. + const std::string rest = safe.substr(i, 32); auto starts = [&](const char* tag) { return rest.rfind(tag, 0) == 0; }; if (starts(tags_.think_open)) { flush_plain(); @@ -207,6 +230,11 @@ class AnswerStream { } void close_call(std::vector>& out) { + if (fam_ == AnswerFamily::gemma4 && close_gemma_call()) { + call_text_.clear(); + mode_ = Mode::content; + return; + } try { const nlohmann::json j = nlohmann::json::parse(call_text_); ToolCall c; @@ -223,6 +251,90 @@ class AnswerStream { mode_ = Mode::content; } + // `call:NAME{key:value,...}` -> a ToolCall whose arguments are JSON, so the two routes hand the caller one shape. + // The Gemma object syntax is not JSON: keys are bare, string values are wrapped in <|"|>, and nested objects and + // arrays use the same bare-key form. This rewrites it rather than guessing: <|"|>v<|"|> becomes a JSON string, + // a bare key gains quotes, and numbers/true/false/null pass through as literals. + // shortcut: values that are neither quoted, numeric, boolean, null nor a brace/bracket nest are emitted as strings, + // re-derive from the pack if a writer starts emitting bare tokens that are meant to be something else. + static std::string gemma_object_to_json(const std::string& body) { + std::string out; + size_t i = 0; + auto space = [&] { while (i < body.size() && (body[i] == ' ' || body[i] == '\n' || body[i] == '\t')) ++i; }; + // one value at `i` + std::function value = [&] { + space(); + if (body.compare(i, 5, "<|\"|>") == 0) { + i += 5; + std::string lit; + while (i < body.size() && body.compare(i, 5, "<|\"|>") != 0) lit += body[i++]; + i += (i < body.size() ? 5 : 0); + out += nlohmann::json(lit).dump(); + return; + } + if (i < body.size() && (body[i] == '{' || body[i] == '[')) { + const char close_ch = body[i] == '{' ? '}' : ']'; + out += body[i++]; + space(); + bool first = true; + while (i < body.size() && body[i] != close_ch) { + if (!first) out += ','; + first = false; + space(); + if (close_ch == '}') { + const size_t colon = body.find(':', i); + std::string key = body.substr(i, colon == std::string::npos ? 0 : colon - i); + while (!key.empty() && (key.back() == ' ' || key.back() == '\n')) key.pop_back(); + out += nlohmann::json(key).dump() + ":"; + i = (colon == std::string::npos) ? body.size() : colon + 1; + value(); + } else { + value(); + } + space(); + if (i < body.size() && body[i] == ',') ++i; + } + if (i < body.size()) { out += body[i++]; } + return; + } + std::string raw; + int depth = 0; + while (i < body.size()) { + const char c = body[i]; + if (c == '{' || c == '[') ++depth; + if (c == '}' || c == ']') { if (depth == 0) break; --depth; } + if (depth == 0 && (c == ',' || c == '}' || c == ']')) break; + raw += c; ++i; + } + while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\n')) raw.pop_back(); + if (raw == "true" || raw == "false" || raw == "null" || (!raw.empty() && (std::isdigit((unsigned char)raw[0]) || raw[0] == '-'))) { + out += raw; + } else { + out += nlohmann::json(raw).dump(); + } + }; + value(); + return out; + } + + bool close_gemma_call() { + const std::string body = answer_detail::trim(call_text_); + if (body.compare(0, 5, "call:") != 0) return false; + const size_t brace = body.find('{'); + if (brace == std::string::npos || body.back() != '}') return false; + ToolCall c; + c.id = "call_" + std::to_string(calls_.size()); + c.name = answer_detail::trim(body.substr(5, brace - 5)); + if (c.name.empty()) return false; + try { + c.arguments = gemma_object_to_json(body.substr(brace, body.size() - brace)); + } catch (const std::exception&) { + return false; + } + calls_.push_back(std::move(c)); + return true; + } + AnswerFamily fam_; answer_detail::FamilyTags tags_; bool tools_sent_ = false; diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 28bc24fac..2f21b926d 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -653,6 +653,46 @@ void test_json_schema_refusals() { "grammar and json_schema together are refused"); } + +// Gemma 4's own markup (engine#1779, F10 fault 4), taken from the pack's tokenizer_config.json: the call body is +// `call:NAME{key:value,...}` with string values wrapped in <|"|>, which is not JSON. Asserted on the normalised +// JSON rather than on "a call was found", because the normalisation is the part that can be silently wrong. +void test_gemma4_answer_parser() { + const auto G = AnswerFamily::gemma4; + { + const std::string reply = + "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>,days:3,metric:true}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1, "gemma4: a <|tool_call> body is a call when tools were sent"); + check(a.calls[0].name == "get_weather", "gemma4: the name is what follows call:"); + check(a.calls[0].arguments == R"({"city":"Paris","days":3,"metric":true})", + "gemma4: arguments are rewritten to JSON (escaped strings, numbers, booleans)"); + check(a.content.empty() && a.content_null, "gemma4: a call-only reply has content: null"); + } + { + // A comma inside an escaped string must not split the argument list -- the case a naive splitter gets wrong. + const std::string reply = + "<|tool_call>call:say{text:<|\"|>hello, world<|\"|>,n:2}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == R"({"text":"hello, world","n":2})", + "gemma4: a comma inside <|\"|>...<|\"|> stays inside the string"); + } + { + const std::string reply = "<|tool_call>call:ping{}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == "{}", "gemma4: a call with no arguments is {}"); + const auto without = parse_answer(reply, G, false); + check(without.calls.empty(), "gemma4: the same markup is not a call when no tools were sent"); + check(without.content.find("<|tool_call>") != std::string::npos, + "gemma4: and it stays in content, marker included"); + } + { + const auto a = parse_answer("<|channel>thought\nlet me thinkDone.", G, false); + check(a.reasoning == "\nlet me think", "gemma4: the thought channel is reasoning"); + check(a.content == "Done.", "gemma4: and the reply after it is content"); + } +} + void test_answer_parser() { const auto F = AnswerFamily::qwen3; { // reasoning, then content: the Qwen3 shape this route sees on every answer @@ -979,6 +1019,7 @@ int main() { test_answer_parser(); test_answer_stream(); test_tool_grammar(); + test_gemma4_answer_parser(); test_tool_request_parsing(); test_content_mask(); test_request_parsing(); From a5ea1e372fd951d3426dfa62f49a014956412a59 Mon Sep 17 00:00:00 2001 From: pi-tmplsec Date: Tue, 10 Nov 2026 00:50:43 -0400 Subject: [PATCH 4/6] serve: the degenerate-repetition predicate, with the floor the symptom needs (engine#1779, F10 fault 2) A reply whose whole body is one character repeated is what FastFlowLM 1.0.12's notes call out (Qwen3.5 answering '//////' after the context limit). The floor is four, not eight: engine#1791's detector used \1{7,}, so a server answering exactly the reported six-character string passed it. Predicate only -- callers decide what to do with it, and nothing here reads the request or the context, which is what lets it be tested without a device or a pack. Twelve cases, chosen around both failure directions: the reported symptom and its four-character boundary, and the too-loose direction (a short real answer, a repeated run followed by punctuation, case differences, digits). --- CMakeLists.txt | 5 +++ app/serve_degenerate.h | 47 +++++++++++++++++++++++++ tests/serve_degenerate_test.cpp | 62 +++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 app/serve_degenerate.h create mode 100644 tests/serve_degenerate_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 83e6d4e25..cc5728656 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -379,6 +379,11 @@ add_executable(think_split_test tests/think_split_test.cpp) target_include_directories(think_split_test PRIVATE app) add_test(NAME think_split COMMAND think_split_test) # Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI. +# engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device. +add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp) +target_include_directories(serve_degenerate_test PRIVATE app) +add_test(NAME serve_degenerate COMMAND serve_degenerate_test) + add_executable(http_guard_test tests/http_guard_test.cpp) target_include_directories(http_guard_test PRIVATE app) add_test(NAME http_guard COMMAND http_guard_test) diff --git a/app/serve_degenerate.h b/app/serve_degenerate.h new file mode 100644 index 000000000..70d8fa741 --- /dev/null +++ b/app/serve_degenerate.h @@ -0,0 +1,47 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// A reply that is one repeated character after the context limit (engine#1779, F10 fault 2). +// +// FastFlowLM 1.0.12's notes list "Qwen3.5 answering `//////` after hitting the context limit". The rule here is deliberately +// blunt, because the thing it detects is blunt: a reply whose whole body is one character repeated. Four is the floor -- +// `//////` (the reported symptom) is six, and an earlier detector in this tree used `\1{7,}`, i.e. eight, so a server +// answering exactly the reported string passed it (engine#1791, closed; the rule below is pi-hrx-sched's, credited). +// +// It is a predicate, not a policy: callers decide what to do (answer with a refusal, close the stream cleanly), and nothing +// here reads the request, the model or the context. That keeps it testable without a device or a pack -- see +// tests/serve_degenerate_test.cpp. +#ifndef ONEBIT_SERVE_DEGENERATE_H +#define ONEBIT_SERVE_DEGENERATE_H + +#include +#include + +namespace onebit { + +// True when `core` (the reply text with leading/trailing whitespace removed) is at least four characters and every one of +// them is the same character. An empty or short reply is not degenerate -- "ok" is a legitimate answer, and "\n\n\n\n" is +// trimmed to nothing before it gets here. +inline bool degenerate_reply(const std::string& core) { + if (core.size() < 4) return false; + for (std::size_t i = 1; i < core.size(); ++i) { + if (core[i] != core[0]) return false; + } + return true; +} + +} // namespace onebit + +#endif // ONEBIT_SERVE_DEGENERATE_H diff --git a/tests/serve_degenerate_test.cpp b/tests/serve_degenerate_test.cpp new file mode 100644 index 000000000..34ebcd7dc --- /dev/null +++ b/tests/serve_degenerate_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// engine#1779 (F10 fault 2): the degenerate-repetition predicate. Header only, no device, no pack -- CI runs it in ms. +// +// The cases are chosen around the two ways this predicate can be wrong: too strict (the reported `//////` slips through -- +// which is what a `\1{7,}` rule did in engine#1791, where eight identical characters were required and the symptom is six) +// and too loose (a legitimate short answer, or a long answer that merely *contains* a repeated run, gets flagged). +#include "serve_degenerate.h" + +#include +#include + +static int failures = 0; + +static void expect(bool want, const std::string& core, const char* why) { + const bool got = onebit::degenerate_reply(core); + if (got != want) { + ++failures; + std::printf("FAIL degenerate_reply(%s) = %s, want %s (%s)\n", core.c_str(), got ? "true" : "false", + want ? "true" : "false", why); + } +} + +int main() { + // The reported symptom, and the reason the floor is four rather than eight. + expect(true, "//////", "the reported symptom: six slashes after the context limit"); + expect(true, "////", "four is the floor and it is inclusive"); + expect(false, "///", "three is short enough to be a real answer"); + expect(false, "", "an empty reply is not degeneracy"); + expect(true, "....", "any character, not just slashes"); + + // Too-loose guards: a real answer must never be flagged. + expect(false, "ok", "a short legitimate answer"); + expect(false, "aaaa!", "a repeated run plus punctuation is a sentence, not degeneracy"); + expect(false, "the cat sat", "ordinary text"); + expect(false, "aa", "two characters"); + + // Boundary: mixed case and digits are different characters. + expect(false, "aAaA", "case matters"); + expect(true, "0000", "digits can degenerate too"); + expect(false, "0001", "and one different character is enough to be fine"); + + if (failures != 0) { + std::printf("%d failure(s)\n", failures); + return 1; + } + std::printf("serve_degenerate: ok\n"); + return 0; +} From 6f8b3e2d162a686883180dae3fcc70fca82500ec Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 10:42:00 -0400 Subject: [PATCH 5/6] npu: gemma4 calls that are malformed are refused, never looped on; tags matched at their own length (engine#1779) Follow-up to engine#1810, which cannot merge: it edits .github/workflows/ci.yml and the merge identity's token has no workflow scope. This carries its four commits unchanged and adds: - serve_degenerate_test is built through add_dependencies(onebit ...), so no workflow edit; - the tag scanner compares at each tag's own length instead of through a 32-byte window; - a gemma4 call body with a value that never arrives (`call:f{a:[1,2}`) looped for ever, appending a comma each pass; it and an unterminated <|"|> string, a key with no value, an unclosed nest and trailing text are now refused and the markup goes back to content; - tests: every family's tags matched whole and split at every byte, the malformed bodies, nested objects and arrays. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 3 +++ app/npu_answer_parser.h | 32 ++++++++++++++++------- tests/npu_sampling_test.cpp | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cc5728656..f4221ac4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -383,6 +383,9 @@ add_test(NAME think_split COMMAND think_split_test) add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp) target_include_directories(serve_degenerate_test PRIVATE app) add_test(NAME serve_degenerate COMMAND serve_degenerate_test) +# Built with `onebit`, which CI's --target list already names: a test target of its own would need a line in +# .github/workflows/ci.yml, and a PR that edits a workflow cannot be merged by a token without the `workflow` scope. +add_dependencies(onebit serve_degenerate_test) add_executable(http_guard_test tests/http_guard_test.cpp) target_include_directories(http_guard_test PRIVATE app) diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index dcd39cffc..0573f5216 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -189,11 +189,13 @@ class AnswerStream { }; while (i < safe.size()) { if (safe[i] == '<') { - // 32 bytes, not 16: gemma4's thinking tag is `<|channel>thought` (18 bytes) and the old - // 16-byte window could never match it -- every tag longer than the window looked like text - // (engine#1779). 32 covers the longest tag any family carries with room to spare. - const std::string rest = safe.substr(i, 32); - auto starts = [&](const char* tag) { return rest.rfind(tag, 0) == 0; }; + // Compared at the tag's own length, never through a fixed window: this read `safe.substr(i, 16)` + // once, and gemma4's thinking tag `<|channel>thought` is 17 bytes, so it could never match -- every + // tag as long as the window read as text (engine#1779). No width is left here to outgrow. + auto starts = [&](const char* tag) { + const size_t n = std::char_traits::length(tag); + return safe.compare(i, n, tag) == 0; + }; if (starts(tags_.think_open)) { flush_plain(); mode_ = Mode::reasoning; @@ -252,6 +254,7 @@ class AnswerStream { } // `call:NAME{key:value,...}` -> a ToolCall whose arguments are JSON, so the two routes hand the caller one shape. + // Throws on a body that is not that shape (the caller puts the markup back into content). // The Gemma object syntax is not JSON: keys are bare, string values are wrapped in <|"|>, and nested objects and // arrays use the same bare-key form. This rewrites it rather than guessing: <|"|>v<|"|> becomes a JSON string, // a bare key gains quotes, and numbers/true/false/null pass through as literals. @@ -268,7 +271,10 @@ class AnswerStream { i += 5; std::string lit; while (i < body.size() && body.compare(i, 5, "<|\"|>") != 0) lit += body[i++]; - i += (i < body.size() ? 5 : 0); + // A string whose closing escape never arrives is malformed, not a shorter string: refusing it sends + // the markup back to content (close_call), where accepting it would invent a truncated argument. + if (i >= body.size()) throw std::runtime_error("gemma4 call: unterminated <|\"|> string"); + i += 5; out += nlohmann::json(lit).dump(); return; } @@ -283,10 +289,12 @@ class AnswerStream { space(); if (close_ch == '}') { const size_t colon = body.find(':', i); - std::string key = body.substr(i, colon == std::string::npos ? 0 : colon - i); + if (colon == std::string::npos) throw std::runtime_error("gemma4 call: a key with no value"); + std::string key = body.substr(i, colon - i); while (!key.empty() && (key.back() == ' ' || key.back() == '\n')) key.pop_back(); + if (key.empty()) throw std::runtime_error("gemma4 call: an empty key"); out += nlohmann::json(key).dump() + ":"; - i = (colon == std::string::npos) ? body.size() : colon + 1; + i = colon + 1; value(); } else { value(); @@ -294,7 +302,8 @@ class AnswerStream { space(); if (i < body.size() && body[i] == ',') ++i; } - if (i < body.size()) { out += body[i++]; } + if (i >= body.size()) throw std::runtime_error("gemma4 call: an object or array that never closes"); + out += body[i++]; return; } std::string raw; @@ -307,6 +316,9 @@ class AnswerStream { raw += c; ++i; } while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\n')) raw.pop_back(); + // An empty value consumed nothing. Refusing it is also what keeps the loops above finite: `[1}` would + // otherwise ask for a value at the `}` for ever, appending a comma each time. + if (raw.empty()) throw std::runtime_error("gemma4 call: an empty value"); if (raw == "true" || raw == "false" || raw == "null" || (!raw.empty() && (std::isdigit((unsigned char)raw[0]) || raw[0] == '-'))) { out += raw; } else { @@ -314,6 +326,8 @@ class AnswerStream { } }; value(); + space(); + if (i != body.size()) throw std::runtime_error("gemma4 call: text after the arguments object"); return out; } diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 2f21b926d..885540fc0 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -691,6 +691,56 @@ void test_gemma4_answer_parser() { check(a.reasoning == "\nlet me think", "gemma4: the thought channel is reasoning"); check(a.content == "Done.", "gemma4: and the reply after it is content"); } + { + // A string whose closing <|"|> never arrives is malformed: no call, and the markup is back in content. + const std::string reply = "<|tool_call>call:say{text:<|\"|>hello}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.empty(), "gemma4: an unterminated <|\"|> string is not a call"); + check(a.content == reply, "gemma4: and what the model wrote is back in content, whole"); + const auto b = parse_answer("<|tool_call>call:f{a:1}tail}", G, true); + check(b.calls.empty() && !b.content.empty(), "gemma4: text after the arguments object is not a call"); + const auto c = parse_answer("<|tool_call>call:f{a:[1,2}", G, true); + check(c.calls.empty() && !c.content.empty(), "gemma4: an array that never closes is not a call"); + } + { + // Nested object and array, the same bare-key form one level down. + const auto a = parse_answer( + "<|tool_call>call:f{o:{k:<|\"|>v<|\"|>},l:[1,<|\"|>a,b<|\"|>],z:null}", G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == R"({"o":{"k":"v"},"l":[1,"a,b"],"z":null})", + "gemma4: nested objects and arrays are rewritten too"); + } +} + +// Every tag every family declares must be recognised, whatever its length, whole and split across two chunks at +// every byte. The scanner once read tags through a 16-byte window, so a 17-byte tag read as text (engine#1779); +// this arm fails for any family whose tags the scanner cannot match, including one added later. +void test_every_family_tag_is_matched() { + for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4}) { + const auto t = onebit::npu::answer_detail::tags_of(f); + const std::string name = onebit::npu::answer_detail::family_name(f); + const std::string think = std::string(t.think_open) + "R" + t.think_close + "C"; + const std::string call = std::string(t.call_open) + "not a call" + t.call_close; + for (size_t cut = 0; cut <= think.size(); cut++) { + AnswerStream s(f, false); + std::vector> out; + s.feed(think.substr(0, cut), out); + s.feed(think.substr(cut), out); + s.finish(out); + check(s.reasoning() == "R" && s.content() == "C", + (name + ": think tags are matched, split at byte " + std::to_string(cut)).c_str()); + } + for (size_t cut = 0; cut <= call.size(); cut++) { + AnswerStream s(f, true); + std::vector> out; + s.feed(call.substr(0, cut), out); + s.feed(call.substr(cut), out); + s.finish(out); + // Not a valid body, so it comes back as content: open tag, body and close tag, which proves both + // call tags were seen (an unseen close tag would leave the body unterminated and drop the tag). + check(s.calls().empty() && s.content() == call && s.reasoning().empty(), + (name + ": call tags are matched, split at byte " + std::to_string(cut)).c_str()); + } + } } void test_answer_parser() { @@ -1020,6 +1070,7 @@ int main() { test_answer_stream(); test_tool_grammar(); test_gemma4_answer_parser(); + test_every_family_tag_is_matched(); test_tool_request_parsing(); test_content_mask(); test_request_parsing(); From acc99940b371fe0dc2dc15872cd5b0ff71269d52 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 10:46:15 -0400 Subject: [PATCH 6/6] forward-serve: split the reply by the markup of the template it rendered (engine#1779, F10 fault 4) The gemma4 answer family had no caller: every container was parsed as qwen3. The family is now read from the container's chat template when that template is what gets rendered, and printed at load. A tool grammar still constrains the reply to qwen3 call markup, so that reply is split as qwen3. Docs in docs/npu-chat-template.md. Co-Authored-By: Claude Fable 5.2 --- app/forward_serve.cpp | 16 +++++++++++++--- app/npu_answer_parser.h | 20 +++++++++++++++++--- docs/npu-chat-template.md | 24 ++++++++++++++++++++++++ tests/npu_sampling_test.cpp | 18 ++++++++++++++++++ 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 2f5b9436d..c70f989e0 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -97,6 +97,9 @@ struct ForwardEngine { // The template the container declares, its sha256 and where it came from: rendered per request // when present (app/npu_template.h), else ChatML when the tokenizer has the anchors. npu::ChatTemplate tmpl; + // The markup this container's replies are split by (app/npu_answer_parser.h): the template's own when + // the template is what gets rendered, else qwen3, which is what the ChatML fallback prompts for. + npu::AnswerFamily answer_family = npu::AnswerFamily::qwen3; std::string bos_text; // the bos/eos *strings* the template's {{ bos_token }} renders std::string eos_text; // decode() of every vocabulary id, built on the first constrained request: the GBNF @@ -534,6 +537,10 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res return json{{"content", content}}; }; + // A tool grammar (tool_choice "required" or a named function) constrains the reply to the qwen3 call + // markup whatever the container is (app/npu_tool_grammar.h), so that reply is split as qwen3. Every + // other reply is in the container's own markup. + const npu::AnswerFamily family = opts.tool_grammar ? npu::AnswerFamily::qwen3 : e.answer_family; auto wrap = [&](const std::string& content, const std::string& finish, const std::vector& lps) { json out; out["id"] = id; @@ -544,7 +551,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // (issue #652 item 3) one shared parser splits what the model wrote into OpenAI's three // parts; a reply that is only tool calls says content: null. const npu::ParsedAnswer ans = - npu::parse_answer(content, npu::AnswerFamily::qwen3, opts.tools_sent); + npu::parse_answer(content, family, opts.tools_sent); json message{{"role", "assistant"}}; message["content"] = ans.content_null ? json(nullptr) : json(ans.content); if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning; @@ -565,7 +572,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res std::vector texts; texts.reserve(lps.size()); for (const auto& x : lps) texts.push_back(x.text); - const std::vector keep = npu::content_mask(texts, npu::AnswerFamily::qwen3, opts.tools_sent); + const std::vector keep = npu::content_mask(texts, family, opts.tools_sent); std::vector content_lps; for (size_t i = 0; i < lps.size(); i++) if (i < keep.size() && keep[i]) content_lps.push_back(lps[i]); @@ -606,7 +613,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res auto chunks = std::make_shared>(); // The same splitter as the non-streamed path, fed piece by piece, so a streamed reply // and a non-streamed one cannot disagree about what is reasoning and what is content. - npu::AnswerStream stream(npu::AnswerFamily::qwen3, opts.tools_sent); + npu::AnswerStream stream(family, opts.tools_sent); auto emit_pieces = [&](const std::string& t, const std::vector& lp) { std::vector> pieces; stream.feed(t, pieces); @@ -933,6 +940,9 @@ int run_forward_serve(int argc, char** argv) { e.tmpl.present() ? e.tmpl.source.c_str() : "no template", e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses"); const bool can_render = e.tmpl.present() && npu::renderer_built(); + if (can_render) e.answer_family = npu::answer_family_of_template(e.tmpl.text); + std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n", + e.answer_family == npu::AnswerFamily::gemma4 ? "gemma4" : "qwen3"); if (e.tmpl.present() && !npu::renderer_built()) std::fprintf(stderr, "1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the " diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 0573f5216..305a6a2c1 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -26,10 +26,11 @@ // model's block is text the caller asked for and stays in `content`; // * a reply that is only tool calls has `content: null` (content_null), not "". // -// Families: Qwen3's markup today -- `…` for reasoning and +// Families: Qwen3's markup -- `…` for reasoning and // `{"name": …, "arguments": {…}}` for calls, which is what this route's ChatML -// models emit. AnswerFamily exists so Gemma 4 (`call:f{…}`) and LFM2 can be added without another -// signature change; an unsupported family is a 400 the caller sees, never a silent wrong split. +// models emit -- and Gemma 4's (`<|channel>thought…`, `<|tool_call>call:f{…}`). +// answer_family_of_template() picks one from the container's own chat template. AnswerFamily exists so +// LFM2 can be added without another signature change. #pragma once #include @@ -124,6 +125,19 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) { } // namespace answer_detail +// Which family a container's replies are written in, read from the chat template the container declares: a +// model writes the markup its template taught it. Gemma 4's template carries `<|tool_call>` and +// `<|channel>thought` as literals (amd/gemma-4-e4b-it's chat_template: one of each); a Qwen3 template carries +// `` and neither of those. No template, or one with neither marker, is qwen3 -- what every +// container was parsed as before this function existed. +inline AnswerFamily answer_family_of_template(const std::string& template_text) { + const answer_detail::FamilyTags g = answer_detail::tags_of(AnswerFamily::gemma4); + if (template_text.find(g.call_open) != std::string::npos || + template_text.find(g.think_open) != std::string::npos) + return AnswerFamily::gemma4; + return AnswerFamily::qwen3; +} + // The incremental splitter. feed() appends whatever is now unambiguous to `out` as // (is_reasoning, text) pieces; finish() flushes what the holdback still held and reports the calls. class AnswerStream { diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index bdf9fb52c..6baf7dac5 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -179,3 +179,27 @@ and the Qwen3 reading of its reply. Stop tokens come from the container's `confi where the model says it does; the parsing of *tool calls* for a non-Qwen3 family is the next piece if a customer packs one. The route-level rows (200 with the container's prompt, 400 for a container with neither) need the device and are recorded in the PR rather than in CI. + +## The reply is split by the template's own markup (engine#1779) + +The template decides what the model was shown, so it also decides what the model writes back. `forward-serve` +reads the answer markup from the template it renders and prints it at load: + +``` +1bit forward-serve: answer markup=gemma4 +``` + +| the rendered template contains | markup | reasoning | a tool call | +|---|---|---|---| +| `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … ` | `<|tool_call>call:NAME{key:value}`, strings as `<|"|>…<|"|>` | +| neither, or no template (ChatML) | `qwen3` | ` … ` | `{"name": …, "arguments": {…}}` | + +Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments` +is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value, +an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged. + +One exception: `tool_choice: "required"` or a named function installs a grammar that constrains the reply to +the `qwen3` call markup for every container (`app/npu_tool_grammar.h`), so that reply is split as `qwen3`. +A grammar in Gemma's own call syntax is not written yet. + +Code: `answer_family_of_template()` in `app/npu_answer_parser.h`; tests in `tests/npu_sampling_test.cpp`. diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 885540fc0..c6877202f 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -711,6 +711,23 @@ void test_gemma4_answer_parser() { } } +// The family comes from the container's chat template. The Gemma 4 lines are the two literals the pack's own +// chat_template carries (amd/gemma-4-e4b-it-mobius-int4_rai_1.9.0_npu: `<|tool_call>` x1, `<|channel>thought` +// x1, `` x0, `` x0), not a guess at them. +void test_answer_family_of_template() { + using onebit::npu::answer_family_of_template; + check(answer_family_of_template("") == AnswerFamily::qwen3, "no template is qwen3, as before"); + check(answer_family_of_template("{%- if tools %}\n{\"name\": ...}{% endif %}") == + AnswerFamily::qwen3, + "a template with and is qwen3"); + check(answer_family_of_template("{{- '<|tool_call>call:' + function['name'] + '{' -}}") == AnswerFamily::gemma4, + "a template that writes <|tool_call>call: is gemma4"); + check(answer_family_of_template("{{- '<|channel>thought\\n' -}}") == AnswerFamily::gemma4, + "a template with the thought channel and no tools is gemma4 too"); + check(answer_family_of_template("<|im_start|>user\n{{ content }}<|im_end|>") == AnswerFamily::qwen3, + "a template with neither marker is qwen3"); +} + // Every tag every family declares must be recognised, whatever its length, whole and split across two chunks at // every byte. The scanner once read tags through a 16-byte window, so a 17-byte tag read as text (engine#1779); // this arm fails for any family whose tags the scanner cannot match, including one added later. @@ -1071,6 +1088,7 @@ int main() { test_tool_grammar(); test_gemma4_answer_parser(); test_every_family_tag_is_matched(); + test_answer_family_of_template(); test_tool_request_parsing(); test_content_mask(); test_request_parsing();