diff --git a/CMakeLists.txt b/CMakeLists.txt index 420dc7928..79a6b1c44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,6 +196,7 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $) # serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU. +add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) @@ -384,6 +385,14 @@ add_executable(think_split_test tests/think_split_test.cpp) target_include_directories(think_split_test PRIVATE app) add_test(NAME think_split COMMAND think_split_test) # Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI. +# engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device. +add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp) +target_include_directories(serve_degenerate_test PRIVATE app) +add_test(NAME serve_degenerate COMMAND serve_degenerate_test) +# Built with `onebit`, which CI's --target list already names: a test target of its own would need a line in +# .github/workflows/ci.yml, and a PR that edits a workflow cannot be merged by a token without the `workflow` scope. +add_dependencies(onebit serve_degenerate_test) + add_executable(http_guard_test tests/http_guard_test.cpp) target_include_directories(http_guard_test PRIVATE app) add_test(NAME http_guard COMMAND http_guard_test) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 2f5b9436d..c70f989e0 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -97,6 +97,9 @@ struct ForwardEngine { // The template the container declares, its sha256 and where it came from: rendered per request // when present (app/npu_template.h), else ChatML when the tokenizer has the anchors. npu::ChatTemplate tmpl; + // The markup this container's replies are split by (app/npu_answer_parser.h): the template's own when + // the template is what gets rendered, else qwen3, which is what the ChatML fallback prompts for. + npu::AnswerFamily answer_family = npu::AnswerFamily::qwen3; std::string bos_text; // the bos/eos *strings* the template's {{ bos_token }} renders std::string eos_text; // decode() of every vocabulary id, built on the first constrained request: the GBNF @@ -534,6 +537,10 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res return json{{"content", content}}; }; + // A tool grammar (tool_choice "required" or a named function) constrains the reply to the qwen3 call + // markup whatever the container is (app/npu_tool_grammar.h), so that reply is split as qwen3. Every + // other reply is in the container's own markup. + const npu::AnswerFamily family = opts.tool_grammar ? npu::AnswerFamily::qwen3 : e.answer_family; auto wrap = [&](const std::string& content, const std::string& finish, const std::vector& lps) { json out; out["id"] = id; @@ -544,7 +551,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // (issue #652 item 3) one shared parser splits what the model wrote into OpenAI's three // parts; a reply that is only tool calls says content: null. const npu::ParsedAnswer ans = - npu::parse_answer(content, npu::AnswerFamily::qwen3, opts.tools_sent); + npu::parse_answer(content, family, opts.tools_sent); json message{{"role", "assistant"}}; message["content"] = ans.content_null ? json(nullptr) : json(ans.content); if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning; @@ -565,7 +572,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res std::vector texts; texts.reserve(lps.size()); for (const auto& x : lps) texts.push_back(x.text); - const std::vector keep = npu::content_mask(texts, npu::AnswerFamily::qwen3, opts.tools_sent); + const std::vector keep = npu::content_mask(texts, family, opts.tools_sent); std::vector content_lps; for (size_t i = 0; i < lps.size(); i++) if (i < keep.size() && keep[i]) content_lps.push_back(lps[i]); @@ -606,7 +613,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res auto chunks = std::make_shared>(); // The same splitter as the non-streamed path, fed piece by piece, so a streamed reply // and a non-streamed one cannot disagree about what is reasoning and what is content. - npu::AnswerStream stream(npu::AnswerFamily::qwen3, opts.tools_sent); + npu::AnswerStream stream(family, opts.tools_sent); auto emit_pieces = [&](const std::string& t, const std::vector& lp) { std::vector> pieces; stream.feed(t, pieces); @@ -933,6 +940,9 @@ int run_forward_serve(int argc, char** argv) { e.tmpl.present() ? e.tmpl.source.c_str() : "no template", e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses"); const bool can_render = e.tmpl.present() && npu::renderer_built(); + if (can_render) e.answer_family = npu::answer_family_of_template(e.tmpl.text); + std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n", + e.answer_family == npu::AnswerFamily::gemma4 ? "gemma4" : "qwen3"); if (e.tmpl.present() && !npu::renderer_built()) std::fprintf(stderr, "1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the " diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 24bc88ef6..305a6a2c1 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -26,14 +26,18 @@ // model's block is text the caller asked for and stays in `content`; // * a reply that is only tool calls has `content: null` (content_null), not "". // -// Families: Qwen3's markup today -- `…` for reasoning and +// Families: Qwen3's markup -- `…` for reasoning and // `{"name": …, "arguments": {…}}` for calls, which is what this route's ChatML -// models emit. AnswerFamily exists so Gemma 4 (`call:f{…}`) and LFM2 can be added without another -// signature change; an unsupported family is a 400 the caller sees, never a silent wrong split. +// models emit -- and Gemma 4's (`<|channel>thought…`, `<|tool_call>call:f{…}`). +// answer_family_of_template() picks one from the container's own chat template. AnswerFamily exists so +// LFM2 can be added without another signature change. #pragma once #include +#include +#include + #include #include #include @@ -44,6 +48,11 @@ namespace onebit::npu { enum class AnswerFamily { qwen3, + // Gemma 4 (engine#1779, F10 fault 4): the pack's own tokenizer_config.json states this markup, and it is not + // qwen3's. Tokens: <|turn>/, <|channel>/, <|tool_call>/, escape <|"|>. + // A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is + // why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows). + gemma4, }; // One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is @@ -63,9 +72,17 @@ struct ParsedAnswer { namespace answer_detail { +inline std::string trim(const std::string& s) { + size_t b = 0, e = s.size(); + while (b < e && (s[b] == ' ' || s[b] == '\n' || s[b] == '\r' || s[b] == '\t')) ++b; + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\n' || s[e - 1] == '\r' || s[e - 1] == '\t')) --e; + return s.substr(b, e - b); +} + inline const char* family_name(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return "qwen3"; + case AnswerFamily::gemma4: return "gemma4"; } return "?"; } @@ -82,6 +99,10 @@ inline FamilyTags tags_of(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return FamilyTags{"", "", "", ""}; + case AnswerFamily::gemma4: + // The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the + // template emits exactly `<|channel>thought\n…\n`, so the pair is the tag. + return FamilyTags{"<|channel>thought", "", "<|tool_call>", ""}; } throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f)); } @@ -104,6 +125,19 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) { } // namespace answer_detail +// Which family a container's replies are written in, read from the chat template the container declares: a +// model writes the markup its template taught it. Gemma 4's template carries `<|tool_call>` and +// `<|channel>thought` as literals (amd/gemma-4-e4b-it's chat_template: one of each); a Qwen3 template carries +// `` and neither of those. No template, or one with neither marker, is qwen3 -- what every +// container was parsed as before this function existed. +inline AnswerFamily answer_family_of_template(const std::string& template_text) { + const answer_detail::FamilyTags g = answer_detail::tags_of(AnswerFamily::gemma4); + if (template_text.find(g.call_open) != std::string::npos || + template_text.find(g.think_open) != std::string::npos) + return AnswerFamily::gemma4; + return AnswerFamily::qwen3; +} + // The incremental splitter. feed() appends whatever is now unambiguous to `out` as // (is_reasoning, text) pieces; finish() flushes what the holdback still held and reports the calls. class AnswerStream { @@ -169,8 +203,13 @@ class AnswerStream { }; while (i < safe.size()) { if (safe[i] == '<') { - const std::string rest = safe.substr(i, 16); - auto starts = [&](const char* tag) { return rest.rfind(tag, 0) == 0; }; + // Compared at the tag's own length, never through a fixed window: this read `safe.substr(i, 16)` + // once, and gemma4's thinking tag `<|channel>thought` is 17 bytes, so it could never match -- every + // tag as long as the window read as text (engine#1779). No width is left here to outgrow. + auto starts = [&](const char* tag) { + const size_t n = std::char_traits::length(tag); + return safe.compare(i, n, tag) == 0; + }; if (starts(tags_.think_open)) { flush_plain(); mode_ = Mode::reasoning; @@ -207,6 +246,11 @@ class AnswerStream { } void close_call(std::vector>& out) { + if (fam_ == AnswerFamily::gemma4 && close_gemma_call()) { + call_text_.clear(); + mode_ = Mode::content; + return; + } try { const nlohmann::json j = nlohmann::json::parse(call_text_); ToolCall c; @@ -223,6 +267,102 @@ class AnswerStream { mode_ = Mode::content; } + // `call:NAME{key:value,...}` -> a ToolCall whose arguments are JSON, so the two routes hand the caller one shape. + // Throws on a body that is not that shape (the caller puts the markup back into content). + // The Gemma object syntax is not JSON: keys are bare, string values are wrapped in <|"|>, and nested objects and + // arrays use the same bare-key form. This rewrites it rather than guessing: <|"|>v<|"|> becomes a JSON string, + // a bare key gains quotes, and numbers/true/false/null pass through as literals. + // shortcut: values that are neither quoted, numeric, boolean, null nor a brace/bracket nest are emitted as strings, + // re-derive from the pack if a writer starts emitting bare tokens that are meant to be something else. + static std::string gemma_object_to_json(const std::string& body) { + std::string out; + size_t i = 0; + auto space = [&] { while (i < body.size() && (body[i] == ' ' || body[i] == '\n' || body[i] == '\t')) ++i; }; + // one value at `i` + std::function value = [&] { + space(); + if (body.compare(i, 5, "<|\"|>") == 0) { + i += 5; + std::string lit; + while (i < body.size() && body.compare(i, 5, "<|\"|>") != 0) lit += body[i++]; + // A string whose closing escape never arrives is malformed, not a shorter string: refusing it sends + // the markup back to content (close_call), where accepting it would invent a truncated argument. + if (i >= body.size()) throw std::runtime_error("gemma4 call: unterminated <|\"|> string"); + i += 5; + out += nlohmann::json(lit).dump(); + return; + } + if (i < body.size() && (body[i] == '{' || body[i] == '[')) { + const char close_ch = body[i] == '{' ? '}' : ']'; + out += body[i++]; + space(); + bool first = true; + while (i < body.size() && body[i] != close_ch) { + if (!first) out += ','; + first = false; + space(); + if (close_ch == '}') { + const size_t colon = body.find(':', i); + if (colon == std::string::npos) throw std::runtime_error("gemma4 call: a key with no value"); + std::string key = body.substr(i, colon - i); + while (!key.empty() && (key.back() == ' ' || key.back() == '\n')) key.pop_back(); + if (key.empty()) throw std::runtime_error("gemma4 call: an empty key"); + out += nlohmann::json(key).dump() + ":"; + i = colon + 1; + value(); + } else { + value(); + } + space(); + if (i < body.size() && body[i] == ',') ++i; + } + if (i >= body.size()) throw std::runtime_error("gemma4 call: an object or array that never closes"); + out += body[i++]; + return; + } + std::string raw; + int depth = 0; + while (i < body.size()) { + const char c = body[i]; + if (c == '{' || c == '[') ++depth; + if (c == '}' || c == ']') { if (depth == 0) break; --depth; } + if (depth == 0 && (c == ',' || c == '}' || c == ']')) break; + raw += c; ++i; + } + while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\n')) raw.pop_back(); + // An empty value consumed nothing. Refusing it is also what keeps the loops above finite: `[1}` would + // otherwise ask for a value at the `}` for ever, appending a comma each time. + if (raw.empty()) throw std::runtime_error("gemma4 call: an empty value"); + if (raw == "true" || raw == "false" || raw == "null" || (!raw.empty() && (std::isdigit((unsigned char)raw[0]) || raw[0] == '-'))) { + out += raw; + } else { + out += nlohmann::json(raw).dump(); + } + }; + value(); + space(); + if (i != body.size()) throw std::runtime_error("gemma4 call: text after the arguments object"); + return out; + } + + bool close_gemma_call() { + const std::string body = answer_detail::trim(call_text_); + if (body.compare(0, 5, "call:") != 0) return false; + const size_t brace = body.find('{'); + if (brace == std::string::npos || body.back() != '}') return false; + ToolCall c; + c.id = "call_" + std::to_string(calls_.size()); + c.name = answer_detail::trim(body.substr(5, brace - 5)); + if (c.name.empty()) return false; + try { + c.arguments = gemma_object_to_json(body.substr(brace, body.size() - brace)); + } catch (const std::exception&) { + return false; + } + calls_.push_back(std::move(c)); + return true; + } + AnswerFamily fam_; answer_detail::FamilyTags tags_; bool tools_sent_ = false; diff --git a/app/serve_degenerate.h b/app/serve_degenerate.h new file mode 100644 index 000000000..70d8fa741 --- /dev/null +++ b/app/serve_degenerate.h @@ -0,0 +1,47 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// A reply that is one repeated character after the context limit (engine#1779, F10 fault 2). +// +// FastFlowLM 1.0.12's notes list "Qwen3.5 answering `//////` after hitting the context limit". The rule here is deliberately +// blunt, because the thing it detects is blunt: a reply whose whole body is one character repeated. Four is the floor -- +// `//////` (the reported symptom) is six, and an earlier detector in this tree used `\1{7,}`, i.e. eight, so a server +// answering exactly the reported string passed it (engine#1791, closed; the rule below is pi-hrx-sched's, credited). +// +// It is a predicate, not a policy: callers decide what to do (answer with a refusal, close the stream cleanly), and nothing +// here reads the request, the model or the context. That keeps it testable without a device or a pack -- see +// tests/serve_degenerate_test.cpp. +#ifndef ONEBIT_SERVE_DEGENERATE_H +#define ONEBIT_SERVE_DEGENERATE_H + +#include +#include + +namespace onebit { + +// True when `core` (the reply text with leading/trailing whitespace removed) is at least four characters and every one of +// them is the same character. An empty or short reply is not degenerate -- "ok" is a legitimate answer, and "\n\n\n\n" is +// trimmed to nothing before it gets here. +inline bool degenerate_reply(const std::string& core) { + if (core.size() < 4) return false; + for (std::size_t i = 1; i < core.size(); ++i) { + if (core[i] != core[0]) return false; + } + return true; +} + +} // namespace onebit + +#endif // ONEBIT_SERVE_DEGENERATE_H diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index bdf9fb52c..6baf7dac5 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -179,3 +179,27 @@ and the Qwen3 reading of its reply. Stop tokens come from the container's `confi where the model says it does; the parsing of *tool calls* for a non-Qwen3 family is the next piece if a customer packs one. The route-level rows (200 with the container's prompt, 400 for a container with neither) need the device and are recorded in the PR rather than in CI. + +## The reply is split by the template's own markup (engine#1779) + +The template decides what the model was shown, so it also decides what the model writes back. `forward-serve` +reads the answer markup from the template it renders and prints it at load: + +``` +1bit forward-serve: answer markup=gemma4 +``` + +| the rendered template contains | markup | reasoning | a tool call | +|---|---|---|---| +| `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … ` | `<|tool_call>call:NAME{key:value}`, strings as `<|"|>…<|"|>` | +| neither, or no template (ChatML) | `qwen3` | ` … ` | `{"name": …, "arguments": {…}}` | + +Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments` +is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value, +an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged. + +One exception: `tool_choice: "required"` or a named function installs a grammar that constrains the reply to +the `qwen3` call markup for every container (`app/npu_tool_grammar.h`), so that reply is split as `qwen3`. +A grammar in Gemma's own call syntax is not written yet. + +Code: `answer_family_of_template()` in `app/npu_answer_parser.h`; tests in `tests/npu_sampling_test.cpp`. diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 91a054cbe..94e6efdf8 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -752,6 +752,113 @@ void test_json_schema_refusals() { "grammar and json_schema together are refused"); } + +// Gemma 4's own markup (engine#1779, F10 fault 4), taken from the pack's tokenizer_config.json: the call body is +// `call:NAME{key:value,...}` with string values wrapped in <|"|>, which is not JSON. Asserted on the normalised +// JSON rather than on "a call was found", because the normalisation is the part that can be silently wrong. +void test_gemma4_answer_parser() { + const auto G = AnswerFamily::gemma4; + { + const std::string reply = + "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>,days:3,metric:true}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1, "gemma4: a <|tool_call> body is a call when tools were sent"); + check(a.calls[0].name == "get_weather", "gemma4: the name is what follows call:"); + check(a.calls[0].arguments == R"({"city":"Paris","days":3,"metric":true})", + "gemma4: arguments are rewritten to JSON (escaped strings, numbers, booleans)"); + check(a.content.empty() && a.content_null, "gemma4: a call-only reply has content: null"); + } + { + // A comma inside an escaped string must not split the argument list -- the case a naive splitter gets wrong. + const std::string reply = + "<|tool_call>call:say{text:<|\"|>hello, world<|\"|>,n:2}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == R"({"text":"hello, world","n":2})", + "gemma4: a comma inside <|\"|>...<|\"|> stays inside the string"); + } + { + const std::string reply = "<|tool_call>call:ping{}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == "{}", "gemma4: a call with no arguments is {}"); + const auto without = parse_answer(reply, G, false); + check(without.calls.empty(), "gemma4: the same markup is not a call when no tools were sent"); + check(without.content.find("<|tool_call>") != std::string::npos, + "gemma4: and it stays in content, marker included"); + } + { + const auto a = parse_answer("<|channel>thought\nlet me thinkDone.", G, false); + check(a.reasoning == "\nlet me think", "gemma4: the thought channel is reasoning"); + check(a.content == "Done.", "gemma4: and the reply after it is content"); + } + { + // A string whose closing <|"|> never arrives is malformed: no call, and the markup is back in content. + const std::string reply = "<|tool_call>call:say{text:<|\"|>hello}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.empty(), "gemma4: an unterminated <|\"|> string is not a call"); + check(a.content == reply, "gemma4: and what the model wrote is back in content, whole"); + const auto b = parse_answer("<|tool_call>call:f{a:1}tail}", G, true); + check(b.calls.empty() && !b.content.empty(), "gemma4: text after the arguments object is not a call"); + const auto c = parse_answer("<|tool_call>call:f{a:[1,2}", G, true); + check(c.calls.empty() && !c.content.empty(), "gemma4: an array that never closes is not a call"); + } + { + // Nested object and array, the same bare-key form one level down. + const auto a = parse_answer( + "<|tool_call>call:f{o:{k:<|\"|>v<|\"|>},l:[1,<|\"|>a,b<|\"|>],z:null}", G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == R"({"o":{"k":"v"},"l":[1,"a,b"],"z":null})", + "gemma4: nested objects and arrays are rewritten too"); + } +} + +// The family comes from the container's chat template. The Gemma 4 lines are the two literals the pack's own +// chat_template carries (amd/gemma-4-e4b-it-mobius-int4_rai_1.9.0_npu: `<|tool_call>` x1, `<|channel>thought` +// x1, `` x0, `` x0), not a guess at them. +void test_answer_family_of_template() { + using onebit::npu::answer_family_of_template; + check(answer_family_of_template("") == AnswerFamily::qwen3, "no template is qwen3, as before"); + check(answer_family_of_template("{%- if tools %}\n{\"name\": ...}{% endif %}") == + AnswerFamily::qwen3, + "a template with and is qwen3"); + check(answer_family_of_template("{{- '<|tool_call>call:' + function['name'] + '{' -}}") == AnswerFamily::gemma4, + "a template that writes <|tool_call>call: is gemma4"); + check(answer_family_of_template("{{- '<|channel>thought\\n' -}}") == AnswerFamily::gemma4, + "a template with the thought channel and no tools is gemma4 too"); + check(answer_family_of_template("<|im_start|>user\n{{ content }}<|im_end|>") == AnswerFamily::qwen3, + "a template with neither marker is qwen3"); +} + +// Every tag every family declares must be recognised, whatever its length, whole and split across two chunks at +// every byte. The scanner once read tags through a 16-byte window, so a 17-byte tag read as text (engine#1779); +// this arm fails for any family whose tags the scanner cannot match, including one added later. +void test_every_family_tag_is_matched() { + for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4}) { + const auto t = onebit::npu::answer_detail::tags_of(f); + const std::string name = onebit::npu::answer_detail::family_name(f); + const std::string think = std::string(t.think_open) + "R" + t.think_close + "C"; + const std::string call = std::string(t.call_open) + "not a call" + t.call_close; + for (size_t cut = 0; cut <= think.size(); cut++) { + AnswerStream s(f, false); + std::vector> out; + s.feed(think.substr(0, cut), out); + s.feed(think.substr(cut), out); + s.finish(out); + check(s.reasoning() == "R" && s.content() == "C", + (name + ": think tags are matched, split at byte " + std::to_string(cut)).c_str()); + } + for (size_t cut = 0; cut <= call.size(); cut++) { + AnswerStream s(f, true); + std::vector> out; + s.feed(call.substr(0, cut), out); + s.feed(call.substr(cut), out); + s.finish(out); + // Not a valid body, so it comes back as content: open tag, body and close tag, which proves both + // call tags were seen (an unseen close tag would leave the body unterminated and drop the tag). + check(s.calls().empty() && s.content() == call && s.reasoning().empty(), + (name + ": call tags are matched, split at byte " + std::to_string(cut)).c_str()); + } + } +} + void test_answer_parser() { const auto F = AnswerFamily::qwen3; { // reasoning, then content: the Qwen3 shape this route sees on every answer @@ -1079,6 +1186,9 @@ int main() { test_answer_parser(); test_answer_stream(); test_tool_grammar(); + test_gemma4_answer_parser(); + test_every_family_tag_is_matched(); + test_answer_family_of_template(); test_tool_request_parsing(); test_content_mask(); test_request_parsing(); diff --git a/tests/serve_degenerate_test.cpp b/tests/serve_degenerate_test.cpp new file mode 100644 index 000000000..34ebcd7dc --- /dev/null +++ b/tests/serve_degenerate_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// engine#1779 (F10 fault 2): the degenerate-repetition predicate. Header only, no device, no pack -- CI runs it in ms. +// +// The cases are chosen around the two ways this predicate can be wrong: too strict (the reported `//////` slips through -- +// which is what a `\1{7,}` rule did in engine#1791, where eight identical characters were required and the symptom is six) +// and too loose (a legitimate short answer, or a long answer that merely *contains* a repeated run, gets flagged). +#include "serve_degenerate.h" + +#include +#include + +static int failures = 0; + +static void expect(bool want, const std::string& core, const char* why) { + const bool got = onebit::degenerate_reply(core); + if (got != want) { + ++failures; + std::printf("FAIL degenerate_reply(%s) = %s, want %s (%s)\n", core.c_str(), got ? "true" : "false", + want ? "true" : "false", why); + } +} + +int main() { + // The reported symptom, and the reason the floor is four rather than eight. + expect(true, "//////", "the reported symptom: six slashes after the context limit"); + expect(true, "////", "four is the floor and it is inclusive"); + expect(false, "///", "three is short enough to be a real answer"); + expect(false, "", "an empty reply is not degeneracy"); + expect(true, "....", "any character, not just slashes"); + + // Too-loose guards: a real answer must never be flagged. + expect(false, "ok", "a short legitimate answer"); + expect(false, "aaaa!", "a repeated run plus punctuation is a sentence, not degeneracy"); + expect(false, "the cat sat", "ordinary text"); + expect(false, "aa", "two characters"); + + // Boundary: mixed case and digits are different characters. + expect(false, "aAaA", "case matters"); + expect(true, "0000", "digits can degenerate too"); + expect(false, "0001", "and one different character is enough to be fine"); + + if (failures != 0) { + std::printf("%d failure(s)\n", failures); + return 1; + } + std::printf("serve_degenerate: ok\n"); + return 0; +} diff --git a/tests/serve_hangup_test.sh b/tests/serve_hangup_test.sh new file mode 100755 index 000000000..70817cd56 --- /dev/null +++ b/tests/serve_hangup_test.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A client that hangs up mid-response must not take the server's accept loop with it (engine #1779, F10 fault 1). +# +# FastFlowLM 1.0.12's notes list "a server that stopped accepting connections after clients hung up" among their fixes. +# The fault is invisible to an ordinary test, because it needs a disconnect *during* a response: the listener is fine, +# the connection table is what leaks or the loop is what exits. So: a request is started against the streaming route, +# the client is killed after it has read a few bytes, and the loop repeats; then a fresh client must be served and the +# process must still be alive. +# +# No GPU, no model: tests/fake_backend.py stands in for the backend (app/serve.cpp only proxies to it), and the port is +# read back from serve's own line rather than picked and released here (tests/serve_port.sh, engine #1485). +# +# usage: tests/serve_hangup_test.sh path/to/1bit +set -uo pipefail + +bin=${1:?usage: serve_hangup_test.sh path/to/1bit} +here=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=tests/serve_port.sh +. "$here/serve_port.sh" +scratch=$(mktemp -d) +touch "$scratch/tiny.gguf" +pids=() +cleanup() { + for p in "${pids[@]}"; do pkill -9 -P "$p" 2>/dev/null; kill -9 "$p" 2>/dev/null; done + wait 2>/dev/null + # serve keeps a state file naming the port it bound; drop it like tests/metrics_serve.sh does, but only if the + # helper exists -- this test must not fail in cleanup for a reason unrelated to the fault it measures. + if command -v statefile >/dev/null 2>&1; then statefile | xargs -r rm -f; fi + rm -rf "$scratch" +} +trap cleanup EXIT +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } + +"$bin" serve -m "$scratch/tiny.gguf" --device cpu --port 0 --alias hangup \ + --llama-server "$here/fake_backend.py" >"$scratch/serve.log" 2>&1 & +pid=$! +pids+=("$pid") +port=$(serve_port_wait "$scratch/serve.log" "$pid") || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" + +for _ in $(seq 1 100); do code=$(curl -s -o /dev/null -w '%{http_code}' "$api/health"); [ "$code" = 200 ] && break; sleep 0.1; done + +body='{"model":"hangup","messages":[{"role":"user","content":"hello"}],"stream":true}' + +# The hang-up: start a streaming request, let it read a little, kill the client. Five times, because a leak of one +# connection per hang-up survives a single iteration. +for i in $(seq 1 5); do + timeout 0.4 curl -sN "$api/v1/chat/completions" -H 'Content-Type: application/json' -d "$body" >"$scratch/hang$i.txt" 2>/dev/null +done + +# How many hang-ups actually reached the response stage: the client is killed by `timeout`, so a non-empty file means the +# server had begun writing when the socket went away -- which is the moment this fault lives in. Reported, not asserted: +# a run where none reached it proves less than it appears to, and the reader deserves to know which they got. +got=0; for i in $(seq 1 5); do [ -s "$scratch/hang$i.txt" ] && got=$((got + 1)); done +echo "note $got of 5 hang-ups had begun receiving a response before the client was killed" + +check "serve is still running after 5 client hang-ups" "kill -0 $pid 2>/dev/null" +code=$(curl -s -o "$scratch/after.txt" -w '%{http_code}' --max-time 5 "$api/health") +check "health answers after the hang-ups (got $code)" "[ \"$code\" = 200 ]" +code=$(curl -s -o "$scratch/fresh.txt" -w '%{http_code}' --max-time 20 \ + "$api/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model":"hangup","messages":[{"role":"user","content":"hello"}]}') +check "a fresh client is served after the hang-ups (got $code)" "[ \"$code\" = 200 ]" +check "and the fresh response carries a body" "[ -s \"$scratch/fresh.txt\" ]" + +if [ $fail -ne 0 ]; then + echo "--- serve log"; cat "$scratch/serve.log" + echo "--- fresh response"; cat "$scratch/fresh.txt" 2>/dev/null + echo FAIL; exit 1 +fi +echo "all hang-up checks passed"