diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7c8399f8..95c4c314f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,7 +130,8 @@ jobs: npu_model_bounds_test npu_forward_loader_test npu_design_grid_test npu_sampling_test \ npu_flm_head_import_test npu_elf_args_test \ npu_template_test npu_chatml_test \ - tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test -j + tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test \ + serve_degenerate_test -j - name: Test # One hung test must not hold the job for hours: each test gets 10 minutes. run: ctest --test-dir build --output-on-failure --timeout 600 diff --git a/CMakeLists.txt b/CMakeLists.txt index 78503673a..9b2c21792 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,6 +196,7 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $) # serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU. +add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) @@ -330,6 +331,11 @@ add_executable(think_split_test tests/think_split_test.cpp) target_include_directories(think_split_test PRIVATE app) add_test(NAME think_split COMMAND think_split_test) # Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI. +# engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device. +add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp) +target_include_directories(serve_degenerate_test PRIVATE app) +add_test(NAME serve_degenerate COMMAND serve_degenerate_test) + add_executable(http_guard_test tests/http_guard_test.cpp) target_include_directories(http_guard_test PRIVATE app) add_test(NAME http_guard COMMAND http_guard_test) diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 24bc88ef6..dcd39cffc 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -34,6 +34,9 @@ #include +#include +#include + #include #include #include @@ -44,6 +47,11 @@ namespace onebit::npu { enum class AnswerFamily { qwen3, + // Gemma 4 (engine#1779, F10 fault 4): the pack's own tokenizer_config.json states this markup, and it is not + // qwen3's. Tokens: <|turn>/, <|channel>/, <|tool_call>/, escape <|"|>. + // A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is + // why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows). + gemma4, }; // One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is @@ -63,9 +71,17 @@ struct ParsedAnswer { namespace answer_detail { +inline std::string trim(const std::string& s) { + size_t b = 0, e = s.size(); + while (b < e && (s[b] == ' ' || s[b] == '\n' || s[b] == '\r' || s[b] == '\t')) ++b; + while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\n' || s[e - 1] == '\r' || s[e - 1] == '\t')) --e; + return s.substr(b, e - b); +} + inline const char* family_name(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return "qwen3"; + case AnswerFamily::gemma4: return "gemma4"; } return "?"; } @@ -82,6 +98,10 @@ inline FamilyTags tags_of(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return FamilyTags{"", "", "", ""}; + case AnswerFamily::gemma4: + // The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the + // template emits exactly `<|channel>thought\n…\n`, so the pair is the tag. + return FamilyTags{"<|channel>thought", "", "<|tool_call>", ""}; } throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f)); } @@ -169,7 +189,10 @@ class AnswerStream { }; while (i < safe.size()) { if (safe[i] == '<') { - const std::string rest = safe.substr(i, 16); + // 32 bytes, not 16: gemma4's thinking tag is `<|channel>thought` (18 bytes) and the old + // 16-byte window could never match it -- every tag longer than the window looked like text + // (engine#1779). 32 covers the longest tag any family carries with room to spare. + const std::string rest = safe.substr(i, 32); auto starts = [&](const char* tag) { return rest.rfind(tag, 0) == 0; }; if (starts(tags_.think_open)) { flush_plain(); @@ -207,6 +230,11 @@ class AnswerStream { } void close_call(std::vector>& out) { + if (fam_ == AnswerFamily::gemma4 && close_gemma_call()) { + call_text_.clear(); + mode_ = Mode::content; + return; + } try { const nlohmann::json j = nlohmann::json::parse(call_text_); ToolCall c; @@ -223,6 +251,90 @@ class AnswerStream { mode_ = Mode::content; } + // `call:NAME{key:value,...}` -> a ToolCall whose arguments are JSON, so the two routes hand the caller one shape. + // The Gemma object syntax is not JSON: keys are bare, string values are wrapped in <|"|>, and nested objects and + // arrays use the same bare-key form. This rewrites it rather than guessing: <|"|>v<|"|> becomes a JSON string, + // a bare key gains quotes, and numbers/true/false/null pass through as literals. + // shortcut: values that are neither quoted, numeric, boolean, null nor a brace/bracket nest are emitted as strings, + // re-derive from the pack if a writer starts emitting bare tokens that are meant to be something else. + static std::string gemma_object_to_json(const std::string& body) { + std::string out; + size_t i = 0; + auto space = [&] { while (i < body.size() && (body[i] == ' ' || body[i] == '\n' || body[i] == '\t')) ++i; }; + // one value at `i` + std::function value = [&] { + space(); + if (body.compare(i, 5, "<|\"|>") == 0) { + i += 5; + std::string lit; + while (i < body.size() && body.compare(i, 5, "<|\"|>") != 0) lit += body[i++]; + i += (i < body.size() ? 5 : 0); + out += nlohmann::json(lit).dump(); + return; + } + if (i < body.size() && (body[i] == '{' || body[i] == '[')) { + const char close_ch = body[i] == '{' ? '}' : ']'; + out += body[i++]; + space(); + bool first = true; + while (i < body.size() && body[i] != close_ch) { + if (!first) out += ','; + first = false; + space(); + if (close_ch == '}') { + const size_t colon = body.find(':', i); + std::string key = body.substr(i, colon == std::string::npos ? 0 : colon - i); + while (!key.empty() && (key.back() == ' ' || key.back() == '\n')) key.pop_back(); + out += nlohmann::json(key).dump() + ":"; + i = (colon == std::string::npos) ? body.size() : colon + 1; + value(); + } else { + value(); + } + space(); + if (i < body.size() && body[i] == ',') ++i; + } + if (i < body.size()) { out += body[i++]; } + return; + } + std::string raw; + int depth = 0; + while (i < body.size()) { + const char c = body[i]; + if (c == '{' || c == '[') ++depth; + if (c == '}' || c == ']') { if (depth == 0) break; --depth; } + if (depth == 0 && (c == ',' || c == '}' || c == ']')) break; + raw += c; ++i; + } + while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\n')) raw.pop_back(); + if (raw == "true" || raw == "false" || raw == "null" || (!raw.empty() && (std::isdigit((unsigned char)raw[0]) || raw[0] == '-'))) { + out += raw; + } else { + out += nlohmann::json(raw).dump(); + } + }; + value(); + return out; + } + + bool close_gemma_call() { + const std::string body = answer_detail::trim(call_text_); + if (body.compare(0, 5, "call:") != 0) return false; + const size_t brace = body.find('{'); + if (brace == std::string::npos || body.back() != '}') return false; + ToolCall c; + c.id = "call_" + std::to_string(calls_.size()); + c.name = answer_detail::trim(body.substr(5, brace - 5)); + if (c.name.empty()) return false; + try { + c.arguments = gemma_object_to_json(body.substr(brace, body.size() - brace)); + } catch (const std::exception&) { + return false; + } + calls_.push_back(std::move(c)); + return true; + } + AnswerFamily fam_; answer_detail::FamilyTags tags_; bool tools_sent_ = false; diff --git a/app/serve_degenerate.h b/app/serve_degenerate.h new file mode 100644 index 000000000..70d8fa741 --- /dev/null +++ b/app/serve_degenerate.h @@ -0,0 +1,47 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// A reply that is one repeated character after the context limit (engine#1779, F10 fault 2). +// +// FastFlowLM 1.0.12's notes list "Qwen3.5 answering `//////` after hitting the context limit". The rule here is deliberately +// blunt, because the thing it detects is blunt: a reply whose whole body is one character repeated. Four is the floor -- +// `//////` (the reported symptom) is six, and an earlier detector in this tree used `\1{7,}`, i.e. eight, so a server +// answering exactly the reported string passed it (engine#1791, closed; the rule below is pi-hrx-sched's, credited). +// +// It is a predicate, not a policy: callers decide what to do (answer with a refusal, close the stream cleanly), and nothing +// here reads the request, the model or the context. That keeps it testable without a device or a pack -- see +// tests/serve_degenerate_test.cpp. +#ifndef ONEBIT_SERVE_DEGENERATE_H +#define ONEBIT_SERVE_DEGENERATE_H + +#include +#include + +namespace onebit { + +// True when `core` (the reply text with leading/trailing whitespace removed) is at least four characters and every one of +// them is the same character. An empty or short reply is not degenerate -- "ok" is a legitimate answer, and "\n\n\n\n" is +// trimmed to nothing before it gets here. +inline bool degenerate_reply(const std::string& core) { + if (core.size() < 4) return false; + for (std::size_t i = 1; i < core.size(); ++i) { + if (core[i] != core[0]) return false; + } + return true; +} + +} // namespace onebit + +#endif // ONEBIT_SERVE_DEGENERATE_H diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 28bc24fac..2f21b926d 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -653,6 +653,46 @@ void test_json_schema_refusals() { "grammar and json_schema together are refused"); } + +// Gemma 4's own markup (engine#1779, F10 fault 4), taken from the pack's tokenizer_config.json: the call body is +// `call:NAME{key:value,...}` with string values wrapped in <|"|>, which is not JSON. Asserted on the normalised +// JSON rather than on "a call was found", because the normalisation is the part that can be silently wrong. +void test_gemma4_answer_parser() { + const auto G = AnswerFamily::gemma4; + { + const std::string reply = + "<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>,days:3,metric:true}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1, "gemma4: a <|tool_call> body is a call when tools were sent"); + check(a.calls[0].name == "get_weather", "gemma4: the name is what follows call:"); + check(a.calls[0].arguments == R"({"city":"Paris","days":3,"metric":true})", + "gemma4: arguments are rewritten to JSON (escaped strings, numbers, booleans)"); + check(a.content.empty() && a.content_null, "gemma4: a call-only reply has content: null"); + } + { + // A comma inside an escaped string must not split the argument list -- the case a naive splitter gets wrong. + const std::string reply = + "<|tool_call>call:say{text:<|\"|>hello, world<|\"|>,n:2}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == R"({"text":"hello, world","n":2})", + "gemma4: a comma inside <|\"|>...<|\"|> stays inside the string"); + } + { + const std::string reply = "<|tool_call>call:ping{}"; + const auto a = parse_answer(reply, G, true); + check(a.calls.size() == 1 && a.calls[0].arguments == "{}", "gemma4: a call with no arguments is {}"); + const auto without = parse_answer(reply, G, false); + check(without.calls.empty(), "gemma4: the same markup is not a call when no tools were sent"); + check(without.content.find("<|tool_call>") != std::string::npos, + "gemma4: and it stays in content, marker included"); + } + { + const auto a = parse_answer("<|channel>thought\nlet me thinkDone.", G, false); + check(a.reasoning == "\nlet me think", "gemma4: the thought channel is reasoning"); + check(a.content == "Done.", "gemma4: and the reply after it is content"); + } +} + void test_answer_parser() { const auto F = AnswerFamily::qwen3; { // reasoning, then content: the Qwen3 shape this route sees on every answer @@ -979,6 +1019,7 @@ int main() { test_answer_parser(); test_answer_stream(); test_tool_grammar(); + test_gemma4_answer_parser(); test_tool_request_parsing(); test_content_mask(); test_request_parsing(); diff --git a/tests/serve_degenerate_test.cpp b/tests/serve_degenerate_test.cpp new file mode 100644 index 000000000..34ebcd7dc --- /dev/null +++ b/tests/serve_degenerate_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// engine#1779 (F10 fault 2): the degenerate-repetition predicate. Header only, no device, no pack -- CI runs it in ms. +// +// The cases are chosen around the two ways this predicate can be wrong: too strict (the reported `//////` slips through -- +// which is what a `\1{7,}` rule did in engine#1791, where eight identical characters were required and the symptom is six) +// and too loose (a legitimate short answer, or a long answer that merely *contains* a repeated run, gets flagged). +#include "serve_degenerate.h" + +#include +#include + +static int failures = 0; + +static void expect(bool want, const std::string& core, const char* why) { + const bool got = onebit::degenerate_reply(core); + if (got != want) { + ++failures; + std::printf("FAIL degenerate_reply(%s) = %s, want %s (%s)\n", core.c_str(), got ? "true" : "false", + want ? "true" : "false", why); + } +} + +int main() { + // The reported symptom, and the reason the floor is four rather than eight. + expect(true, "//////", "the reported symptom: six slashes after the context limit"); + expect(true, "////", "four is the floor and it is inclusive"); + expect(false, "///", "three is short enough to be a real answer"); + expect(false, "", "an empty reply is not degeneracy"); + expect(true, "....", "any character, not just slashes"); + + // Too-loose guards: a real answer must never be flagged. + expect(false, "ok", "a short legitimate answer"); + expect(false, "aaaa!", "a repeated run plus punctuation is a sentence, not degeneracy"); + expect(false, "the cat sat", "ordinary text"); + expect(false, "aa", "two characters"); + + // Boundary: mixed case and digits are different characters. + expect(false, "aAaA", "case matters"); + expect(true, "0000", "digits can degenerate too"); + expect(false, "0001", "and one different character is enough to be fine"); + + if (failures != 0) { + std::printf("%d failure(s)\n", failures); + return 1; + } + std::printf("serve_degenerate: ok\n"); + return 0; +} diff --git a/tests/serve_hangup_test.sh b/tests/serve_hangup_test.sh new file mode 100755 index 000000000..70817cd56 --- /dev/null +++ b/tests/serve_hangup_test.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A client that hangs up mid-response must not take the server's accept loop with it (engine #1779, F10 fault 1). +# +# FastFlowLM 1.0.12's notes list "a server that stopped accepting connections after clients hung up" among their fixes. +# The fault is invisible to an ordinary test, because it needs a disconnect *during* a response: the listener is fine, +# the connection table is what leaks or the loop is what exits. So: a request is started against the streaming route, +# the client is killed after it has read a few bytes, and the loop repeats; then a fresh client must be served and the +# process must still be alive. +# +# No GPU, no model: tests/fake_backend.py stands in for the backend (app/serve.cpp only proxies to it), and the port is +# read back from serve's own line rather than picked and released here (tests/serve_port.sh, engine #1485). +# +# usage: tests/serve_hangup_test.sh path/to/1bit +set -uo pipefail + +bin=${1:?usage: serve_hangup_test.sh path/to/1bit} +here=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=tests/serve_port.sh +. "$here/serve_port.sh" +scratch=$(mktemp -d) +touch "$scratch/tiny.gguf" +pids=() +cleanup() { + for p in "${pids[@]}"; do pkill -9 -P "$p" 2>/dev/null; kill -9 "$p" 2>/dev/null; done + wait 2>/dev/null + # serve keeps a state file naming the port it bound; drop it like tests/metrics_serve.sh does, but only if the + # helper exists -- this test must not fail in cleanup for a reason unrelated to the fault it measures. + if command -v statefile >/dev/null 2>&1; then statefile | xargs -r rm -f; fi + rm -rf "$scratch" +} +trap cleanup EXIT +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } + +"$bin" serve -m "$scratch/tiny.gguf" --device cpu --port 0 --alias hangup \ + --llama-server "$here/fake_backend.py" >"$scratch/serve.log" 2>&1 & +pid=$! +pids+=("$pid") +port=$(serve_port_wait "$scratch/serve.log" "$pid") || { echo "FAIL serve never reported a port"; cat "$scratch/serve.log"; exit 1; } +api="http://127.0.0.1:$port" + +for _ in $(seq 1 100); do code=$(curl -s -o /dev/null -w '%{http_code}' "$api/health"); [ "$code" = 200 ] && break; sleep 0.1; done + +body='{"model":"hangup","messages":[{"role":"user","content":"hello"}],"stream":true}' + +# The hang-up: start a streaming request, let it read a little, kill the client. Five times, because a leak of one +# connection per hang-up survives a single iteration. +for i in $(seq 1 5); do + timeout 0.4 curl -sN "$api/v1/chat/completions" -H 'Content-Type: application/json' -d "$body" >"$scratch/hang$i.txt" 2>/dev/null +done + +# How many hang-ups actually reached the response stage: the client is killed by `timeout`, so a non-empty file means the +# server had begun writing when the socket went away -- which is the moment this fault lives in. Reported, not asserted: +# a run where none reached it proves less than it appears to, and the reader deserves to know which they got. +got=0; for i in $(seq 1 5); do [ -s "$scratch/hang$i.txt" ] && got=$((got + 1)); done +echo "note $got of 5 hang-ups had begun receiving a response before the client was killed" + +check "serve is still running after 5 client hang-ups" "kill -0 $pid 2>/dev/null" +code=$(curl -s -o "$scratch/after.txt" -w '%{http_code}' --max-time 5 "$api/health") +check "health answers after the hang-ups (got $code)" "[ \"$code\" = 200 ]" +code=$(curl -s -o "$scratch/fresh.txt" -w '%{http_code}' --max-time 20 \ + "$api/v1/chat/completions" -H 'Content-Type: application/json' \ + -d '{"model":"hangup","messages":[{"role":"user","content":"hello"}]}') +check "a fresh client is served after the hang-ups (got $code)" "[ \"$code\" = 200 ]" +check "and the fresh response carries a body" "[ -s \"$scratch/fresh.txt\" ]" + +if [ $fail -ne 0 ]; then + echo "--- serve log"; cat "$scratch/serve.log" + echo "--- fresh response"; cat "$scratch/fresh.txt" 2>/dev/null + echo FAIL; exit 1 +fi +echo "all hang-up checks passed"