Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ jobs:
npu_model_bounds_test npu_forward_loader_test npu_design_grid_test npu_sampling_test \
npu_flm_head_import_test npu_elf_args_test \
npu_template_test npu_chatml_test \
tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test -j
tokenizer_linear_test hrx_placement_test hrx_cwsr_test hrx_runtime_test npu_device_lock_test \
serve_degenerate_test -j
- name: Test
# One hung test must not hold the job for hours: each test gets 10 minutes.
run: ctest --test-dir build --output-on-failure --timeout 600
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $<TAR
# serve takes a hung backend down (docs/serve.md, "When a backend hangs"): a fake backend, no GPU.
add_test(NAME watchdog_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/watchdog_serve.sh $<TARGET_FILE:onebit>)
# serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU.
add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $<TARGET_FILE:onebit>)
add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $<TARGET_FILE:onebit>)
# serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU.
add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $<TARGET_FILE:onebit>)
Expand Down Expand Up @@ -330,6 +331,11 @@ add_executable(think_split_test tests/think_split_test.cpp)
target_include_directories(think_split_test PRIVATE app)
add_test(NAME think_split COMMAND think_split_test)
# Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI.
# engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device.
add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp)
target_include_directories(serve_degenerate_test PRIVATE app)
add_test(NAME serve_degenerate COMMAND serve_degenerate_test)

add_executable(http_guard_test tests/http_guard_test.cpp)
target_include_directories(http_guard_test PRIVATE app)
add_test(NAME http_guard COMMAND http_guard_test)
Expand Down
114 changes: 113 additions & 1 deletion app/npu_answer_parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@

#include <nlohmann/json.hpp>

#include <cctype>
#include <functional>

#include <algorithm>
#include <stdexcept>
#include <string>
Expand All @@ -44,6 +47,11 @@ namespace onebit::npu {

enum class AnswerFamily {
qwen3,
// Gemma 4 (engine#1779, F10 fault 4): the pack's own tokenizer_config.json states this markup, and it is not
// qwen3's. Tokens: <|turn>/<turn|>, <|channel>/<channel|>, <|tool_call>/<tool_call|>, escape <|"|>.
// A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is
// why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows).
gemma4,
};

// One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is
Expand All @@ -63,9 +71,17 @@ struct ParsedAnswer {

namespace answer_detail {

inline std::string trim(const std::string& s) {
size_t b = 0, e = s.size();
while (b < e && (s[b] == ' ' || s[b] == '\n' || s[b] == '\r' || s[b] == '\t')) ++b;
while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\n' || s[e - 1] == '\r' || s[e - 1] == '\t')) --e;
return s.substr(b, e - b);
}

inline const char* family_name(AnswerFamily f) {
switch (f) {
case AnswerFamily::qwen3: return "qwen3";
case AnswerFamily::gemma4: return "gemma4";
}
return "?";
}
Expand All @@ -82,6 +98,10 @@ inline FamilyTags tags_of(AnswerFamily f) {
switch (f) {
case AnswerFamily::qwen3:
return FamilyTags{"<think>", "</think>", "<tool_call>", "</tool_call>"};
case AnswerFamily::gemma4:
// The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the
// template emits exactly `<|channel>thought\n…\n<channel|>`, so the pair is the tag.
return FamilyTags{"<|channel>thought", "<channel|>", "<|tool_call>", "<tool_call|>"};
}
throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f));
}
Expand Down Expand Up @@ -169,7 +189,10 @@ class AnswerStream {
};
while (i < safe.size()) {
if (safe[i] == '<') {
const std::string rest = safe.substr(i, 16);
// 32 bytes, not 16: gemma4's thinking tag is `<|channel>thought` (18 bytes) and the old
// 16-byte window could never match it -- every tag longer than the window looked like text
// (engine#1779). 32 covers the longest tag any family carries with room to spare.
const std::string rest = safe.substr(i, 32);
auto starts = [&](const char* tag) { return rest.rfind(tag, 0) == 0; };
if (starts(tags_.think_open)) {
flush_plain();
Expand Down Expand Up @@ -207,6 +230,11 @@ class AnswerStream {
}

void close_call(std::vector<std::pair<bool, std::string>>& out) {
if (fam_ == AnswerFamily::gemma4 && close_gemma_call()) {
call_text_.clear();
mode_ = Mode::content;
return;
}
try {
const nlohmann::json j = nlohmann::json::parse(call_text_);
ToolCall c;
Expand All @@ -223,6 +251,90 @@ class AnswerStream {
mode_ = Mode::content;
}

// `call:NAME{key:value,...}` -> a ToolCall whose arguments are JSON, so the two routes hand the caller one shape.
// The Gemma object syntax is not JSON: keys are bare, string values are wrapped in <|"|>, and nested objects and
// arrays use the same bare-key form. This rewrites it rather than guessing: <|"|>v<|"|> becomes a JSON string,
// a bare key gains quotes, and numbers/true/false/null pass through as literals.
// shortcut: values that are neither quoted, numeric, boolean, null nor a brace/bracket nest are emitted as strings,
// re-derive from the pack if a writer starts emitting bare tokens that are meant to be something else.
static std::string gemma_object_to_json(const std::string& body) {
std::string out;
size_t i = 0;
auto space = [&] { while (i < body.size() && (body[i] == ' ' || body[i] == '\n' || body[i] == '\t')) ++i; };
// one value at `i`
std::function<void()> value = [&] {
space();
if (body.compare(i, 5, "<|\"|>") == 0) {
i += 5;
std::string lit;
while (i < body.size() && body.compare(i, 5, "<|\"|>") != 0) lit += body[i++];
i += (i < body.size() ? 5 : 0);
out += nlohmann::json(lit).dump();
return;
}
if (i < body.size() && (body[i] == '{' || body[i] == '[')) {
const char close_ch = body[i] == '{' ? '}' : ']';
out += body[i++];
space();
bool first = true;
while (i < body.size() && body[i] != close_ch) {
if (!first) out += ',';
first = false;
space();
if (close_ch == '}') {
const size_t colon = body.find(':', i);
std::string key = body.substr(i, colon == std::string::npos ? 0 : colon - i);
while (!key.empty() && (key.back() == ' ' || key.back() == '\n')) key.pop_back();
out += nlohmann::json(key).dump() + ":";
i = (colon == std::string::npos) ? body.size() : colon + 1;
value();
} else {
value();
}
space();
if (i < body.size() && body[i] == ',') ++i;
}
if (i < body.size()) { out += body[i++]; }
return;
}
std::string raw;
int depth = 0;
while (i < body.size()) {
const char c = body[i];
if (c == '{' || c == '[') ++depth;
if (c == '}' || c == ']') { if (depth == 0) break; --depth; }
if (depth == 0 && (c == ',' || c == '}' || c == ']')) break;
raw += c; ++i;
}
while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\n')) raw.pop_back();
if (raw == "true" || raw == "false" || raw == "null" || (!raw.empty() && (std::isdigit((unsigned char)raw[0]) || raw[0] == '-'))) {
out += raw;
} else {
out += nlohmann::json(raw).dump();
}
};
value();
return out;
}

bool close_gemma_call() {
const std::string body = answer_detail::trim(call_text_);
if (body.compare(0, 5, "call:") != 0) return false;
const size_t brace = body.find('{');
if (brace == std::string::npos || body.back() != '}') return false;
ToolCall c;
c.id = "call_" + std::to_string(calls_.size());
c.name = answer_detail::trim(body.substr(5, brace - 5));
if (c.name.empty()) return false;
try {
c.arguments = gemma_object_to_json(body.substr(brace, body.size() - brace));
} catch (const std::exception&) {
return false;
}
calls_.push_back(std::move(c));
return true;
}

AnswerFamily fam_;
answer_detail::FamilyTags tags_;
bool tools_sent_ = false;
Expand Down
47 changes: 47 additions & 0 deletions app/serve_degenerate.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A reply that is one repeated character after the context limit (engine#1779, F10 fault 2).
//
// FastFlowLM 1.0.12's notes list "Qwen3.5 answering `//////` after hitting the context limit". The rule here is deliberately
// blunt, because the thing it detects is blunt: a reply whose whole body is one character repeated. Four is the floor --
// `//////` (the reported symptom) is six, and an earlier detector in this tree used `\1{7,}`, i.e. eight, so a server
// answering exactly the reported string passed it (engine#1791, closed; the rule below is pi-hrx-sched's, credited).
//
// It is a predicate, not a policy: callers decide what to do (answer with a refusal, close the stream cleanly), and nothing
// here reads the request, the model or the context. That keeps it testable without a device or a pack -- see
// tests/serve_degenerate_test.cpp.
#ifndef ONEBIT_SERVE_DEGENERATE_H
#define ONEBIT_SERVE_DEGENERATE_H

#include <cstddef>
#include <string>

namespace onebit {

// True when `core` (the reply text with leading/trailing whitespace removed) is at least four characters and every one of
// them is the same character. An empty or short reply is not degenerate -- "ok" is a legitimate answer, and "\n\n\n\n" is
// trimmed to nothing before it gets here.
inline bool degenerate_reply(const std::string& core) {
if (core.size() < 4) return false;
for (std::size_t i = 1; i < core.size(); ++i) {
if (core[i] != core[0]) return false;
}
return true;
}

} // namespace onebit

#endif // ONEBIT_SERVE_DEGENERATE_H
41 changes: 41 additions & 0 deletions tests/npu_sampling_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -653,6 +653,46 @@ void test_json_schema_refusals() {
"grammar and json_schema together are refused");
}


// Gemma 4's own markup (engine#1779, F10 fault 4), taken from the pack's tokenizer_config.json: the call body is
// `call:NAME{key:value,...}` with string values wrapped in <|"|>, which is not JSON. Asserted on the normalised
// JSON rather than on "a call was found", because the normalisation is the part that can be silently wrong.
void test_gemma4_answer_parser() {
const auto G = AnswerFamily::gemma4;
{
const std::string reply =
"<|tool_call>call:get_weather{city:<|\"|>Paris<|\"|>,days:3,metric:true}<tool_call|>";
const auto a = parse_answer(reply, G, true);
check(a.calls.size() == 1, "gemma4: a <|tool_call> body is a call when tools were sent");
check(a.calls[0].name == "get_weather", "gemma4: the name is what follows call:");
check(a.calls[0].arguments == R"({"city":"Paris","days":3,"metric":true})",
"gemma4: arguments are rewritten to JSON (escaped strings, numbers, booleans)");
check(a.content.empty() && a.content_null, "gemma4: a call-only reply has content: null");
}
{
// A comma inside an escaped string must not split the argument list -- the case a naive splitter gets wrong.
const std::string reply =
"<|tool_call>call:say{text:<|\"|>hello, world<|\"|>,n:2}<tool_call|>";
const auto a = parse_answer(reply, G, true);
check(a.calls.size() == 1 && a.calls[0].arguments == R"({"text":"hello, world","n":2})",
"gemma4: a comma inside <|\"|>...<|\"|> stays inside the string");
}
{
const std::string reply = "<|tool_call>call:ping{}<tool_call|>";
const auto a = parse_answer(reply, G, true);
check(a.calls.size() == 1 && a.calls[0].arguments == "{}", "gemma4: a call with no arguments is {}");
const auto without = parse_answer(reply, G, false);
check(without.calls.empty(), "gemma4: the same markup is not a call when no tools were sent");
check(without.content.find("<|tool_call>") != std::string::npos,
"gemma4: and it stays in content, marker included");
}
{
const auto a = parse_answer("<|channel>thought\nlet me think<channel|>Done.", G, false);
check(a.reasoning == "\nlet me think", "gemma4: the thought channel is reasoning");
check(a.content == "Done.", "gemma4: and the reply after it is content");
}
}

void test_answer_parser() {
const auto F = AnswerFamily::qwen3;
{ // reasoning, then content: the Qwen3 shape this route sees on every answer
Expand Down Expand Up @@ -979,6 +1019,7 @@ int main() {
test_answer_parser();
test_answer_stream();
test_tool_grammar();
test_gemma4_answer_parser();
test_tool_request_parsing();
test_content_mask();
test_request_parsing();
Expand Down
62 changes: 62 additions & 0 deletions tests/serve_degenerate_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// engine#1779 (F10 fault 2): the degenerate-repetition predicate. Header only, no device, no pack -- CI runs it in ms.
//
// The cases are chosen around the two ways this predicate can be wrong: too strict (the reported `//////` slips through --
// which is what a `\1{7,}` rule did in engine#1791, where eight identical characters were required and the symptom is six)
// and too loose (a legitimate short answer, or a long answer that merely *contains* a repeated run, gets flagged).
#include "serve_degenerate.h"

#include <cstdio>
#include <string>

static int failures = 0;

static void expect(bool want, const std::string& core, const char* why) {
const bool got = onebit::degenerate_reply(core);
if (got != want) {
++failures;
std::printf("FAIL degenerate_reply(%s) = %s, want %s (%s)\n", core.c_str(), got ? "true" : "false",
want ? "true" : "false", why);
}
}

int main() {
// The reported symptom, and the reason the floor is four rather than eight.
expect(true, "//////", "the reported symptom: six slashes after the context limit");
expect(true, "////", "four is the floor and it is inclusive");
expect(false, "///", "three is short enough to be a real answer");
expect(false, "", "an empty reply is not degeneracy");
expect(true, "....", "any character, not just slashes");

// Too-loose guards: a real answer must never be flagged.
expect(false, "ok", "a short legitimate answer");
expect(false, "aaaa!", "a repeated run plus punctuation is a sentence, not degeneracy");
expect(false, "the cat sat", "ordinary text");
expect(false, "aa", "two characters");

// Boundary: mixed case and digits are different characters.
expect(false, "aAaA", "case matters");
expect(true, "0000", "digits can degenerate too");
expect(false, "0001", "and one different character is enough to be fine");

if (failures != 0) {
std::printf("%d failure(s)\n", failures);
return 1;
}
std::printf("serve_degenerate: ok\n");
return 0;
}
Loading
Loading