Skip to content
9 changes: 9 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $<TAR
# serve takes a hung backend down (docs/serve.md, "When a backend hangs"): a fake backend, no GPU.
add_test(NAME watchdog_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/watchdog_serve.sh $<TARGET_FILE:onebit>)
# serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU.
add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $<TARGET_FILE:onebit>)
add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $<TARGET_FILE:onebit>)
# serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU.
add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $<TARGET_FILE:onebit>)
Expand Down Expand Up @@ -384,6 +385,14 @@ add_executable(think_split_test tests/think_split_test.cpp)
target_include_directories(think_split_test PRIVATE app)
add_test(NAME think_split COMMAND think_split_test)
# Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI.
# engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device.
add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp)
target_include_directories(serve_degenerate_test PRIVATE app)
add_test(NAME serve_degenerate COMMAND serve_degenerate_test)
# Built with `onebit`, which CI's --target list already names: a test target of its own would need a line in
# .github/workflows/ci.yml, and a PR that edits a workflow cannot be merged by a token without the `workflow` scope.
add_dependencies(onebit serve_degenerate_test)

add_executable(http_guard_test tests/http_guard_test.cpp)
target_include_directories(http_guard_test PRIVATE app)
add_test(NAME http_guard COMMAND http_guard_test)
Expand Down
16 changes: 13 additions & 3 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ struct ForwardEngine {
// The template the container declares, its sha256 and where it came from: rendered per request
// when present (app/npu_template.h), else ChatML when the tokenizer has the anchors.
npu::ChatTemplate tmpl;
// The markup this container's replies are split by (app/npu_answer_parser.h): the template's own when
// the template is what gets rendered, else qwen3, which is what the ChatML fallback prompts for.
npu::AnswerFamily answer_family = npu::AnswerFamily::qwen3;
std::string bos_text; // the bos/eos *strings* the template's {{ bos_token }} renders
std::string eos_text;
// decode() of every vocabulary id, built on the first constrained request: the GBNF
Expand Down Expand Up @@ -534,6 +537,10 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
return json{{"content", content}};
};

// A tool grammar (tool_choice "required" or a named function) constrains the reply to the qwen3 call
// markup whatever the container is (app/npu_tool_grammar.h), so that reply is split as qwen3. Every
// other reply is in the container's own markup.
const npu::AnswerFamily family = opts.tool_grammar ? npu::AnswerFamily::qwen3 : e.answer_family;
auto wrap = [&](const std::string& content, const std::string& finish, const std::vector<Emitted>& lps) {
json out;
out["id"] = id;
Expand All @@ -544,7 +551,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
// (issue #652 item 3) one shared parser splits what the model wrote into OpenAI's three
// parts; a reply that is only tool calls says content: null.
const npu::ParsedAnswer ans =
npu::parse_answer(content, npu::AnswerFamily::qwen3, opts.tools_sent);
npu::parse_answer(content, family, opts.tools_sent);
json message{{"role", "assistant"}};
message["content"] = ans.content_null ? json(nullptr) : json(ans.content);
if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning;
Expand All @@ -565,7 +572,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
std::vector<std::string> texts;
texts.reserve(lps.size());
for (const auto& x : lps) texts.push_back(x.text);
const std::vector<char> keep = npu::content_mask(texts, npu::AnswerFamily::qwen3, opts.tools_sent);
const std::vector<char> keep = npu::content_mask(texts, family, opts.tools_sent);
std::vector<Emitted> content_lps;
for (size_t i = 0; i < lps.size(); i++)
if (i < keep.size() && keep[i]) content_lps.push_back(lps[i]);
Expand Down Expand Up @@ -606,7 +613,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
auto chunks = std::make_shared<std::vector<std::string>>();
// The same splitter as the non-streamed path, fed piece by piece, so a streamed reply
// and a non-streamed one cannot disagree about what is reasoning and what is content.
npu::AnswerStream stream(npu::AnswerFamily::qwen3, opts.tools_sent);
npu::AnswerStream stream(family, opts.tools_sent);
auto emit_pieces = [&](const std::string& t, const std::vector<Emitted>& lp) {
std::vector<std::pair<bool, std::string>> pieces;
stream.feed(t, pieces);
Expand Down Expand Up @@ -933,6 +940,9 @@ int run_forward_serve(int argc, char** argv) {
e.tmpl.present() ? e.tmpl.source.c_str() : "no template",
e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses");
const bool can_render = e.tmpl.present() && npu::renderer_built();
if (can_render) e.answer_family = npu::answer_family_of_template(e.tmpl.text);
std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n",
e.answer_family == npu::AnswerFamily::gemma4 ? "gemma4" : "qwen3");
if (e.tmpl.present() && !npu::renderer_built())
std::fprintf(stderr,
"1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the "
Expand Down
150 changes: 145 additions & 5 deletions app/npu_answer_parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,18 @@
// model's <tool_call> block is text the caller asked for and stays in `content`;
// * a reply that is only tool calls has `content: null` (content_null), not "".
//
// Families: Qwen3's markup today -- `<think>…</think>` for reasoning and
// Families: Qwen3's markup -- `<think>…</think>` for reasoning and
// `<tool_call>{"name": …, "arguments": {…}}</tool_call>` for calls, which is what this route's ChatML
// models emit. AnswerFamily exists so Gemma 4 (`call:f{…}`) and LFM2 can be added without another
// signature change; an unsupported family is a 400 the caller sees, never a silent wrong split.
// models emit -- and Gemma 4's (`<|channel>thought…<channel|>`, `<|tool_call>call:f{…}<tool_call|>`).
// answer_family_of_template() picks one from the container's own chat template. AnswerFamily exists so
// LFM2 can be added without another signature change.
#pragma once

#include <nlohmann/json.hpp>

#include <cctype>
#include <functional>

#include <algorithm>
#include <stdexcept>
#include <string>
Expand All @@ -44,6 +48,11 @@ namespace onebit::npu {

enum class AnswerFamily {
qwen3,
// Gemma 4 (engine#1779, F10 fault 4): the pack's own tokenizer_config.json states this markup, and it is not
// qwen3's. Tokens: <|turn>/<turn|>, <|channel>/<channel|>, <|tool_call>/<tool_call|>, escape <|"|>.
// A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is
// why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows).
gemma4,
};

// One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is
Expand All @@ -63,9 +72,17 @@ struct ParsedAnswer {

namespace answer_detail {

inline std::string trim(const std::string& s) {
size_t b = 0, e = s.size();
while (b < e && (s[b] == ' ' || s[b] == '\n' || s[b] == '\r' || s[b] == '\t')) ++b;
while (e > b && (s[e - 1] == ' ' || s[e - 1] == '\n' || s[e - 1] == '\r' || s[e - 1] == '\t')) --e;
return s.substr(b, e - b);
}

inline const char* family_name(AnswerFamily f) {
switch (f) {
case AnswerFamily::qwen3: return "qwen3";
case AnswerFamily::gemma4: return "gemma4";
}
return "?";
}
Expand All @@ -82,6 +99,10 @@ inline FamilyTags tags_of(AnswerFamily f) {
switch (f) {
case AnswerFamily::qwen3:
return FamilyTags{"<think>", "</think>", "<tool_call>", "</tool_call>"};
case AnswerFamily::gemma4:
// The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the
// template emits exactly `<|channel>thought\n…\n<channel|>`, so the pair is the tag.
return FamilyTags{"<|channel>thought", "<channel|>", "<|tool_call>", "<tool_call|>"};
}
throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f));
}
Expand All @@ -104,6 +125,19 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) {

} // namespace answer_detail

// Which family a container's replies are written in, read from the chat template the container declares: a
// model writes the markup its template taught it. Gemma 4's template carries `<|tool_call>` and
// `<|channel>thought` as literals (amd/gemma-4-e4b-it's chat_template: one of each); a Qwen3 template carries
// `<tool_call>` and neither of those. No template, or one with neither marker, is qwen3 -- what every
// container was parsed as before this function existed.
inline AnswerFamily answer_family_of_template(const std::string& template_text) {
const answer_detail::FamilyTags g = answer_detail::tags_of(AnswerFamily::gemma4);
if (template_text.find(g.call_open) != std::string::npos ||
template_text.find(g.think_open) != std::string::npos)
return AnswerFamily::gemma4;
return AnswerFamily::qwen3;
}

// The incremental splitter. feed() appends whatever is now unambiguous to `out` as
// (is_reasoning, text) pieces; finish() flushes what the holdback still held and reports the calls.
class AnswerStream {
Expand Down Expand Up @@ -169,8 +203,13 @@ class AnswerStream {
};
while (i < safe.size()) {
if (safe[i] == '<') {
const std::string rest = safe.substr(i, 16);
auto starts = [&](const char* tag) { return rest.rfind(tag, 0) == 0; };
// Compared at the tag's own length, never through a fixed window: this read `safe.substr(i, 16)`
// once, and gemma4's thinking tag `<|channel>thought` is 17 bytes, so it could never match -- every
// tag as long as the window read as text (engine#1779). No width is left here to outgrow.
auto starts = [&](const char* tag) {
const size_t n = std::char_traits<char>::length(tag);
return safe.compare(i, n, tag) == 0;
};
if (starts(tags_.think_open)) {
flush_plain();
mode_ = Mode::reasoning;
Expand Down Expand Up @@ -207,6 +246,11 @@ class AnswerStream {
}

void close_call(std::vector<std::pair<bool, std::string>>& out) {
if (fam_ == AnswerFamily::gemma4 && close_gemma_call()) {
call_text_.clear();
mode_ = Mode::content;
return;
}
try {
const nlohmann::json j = nlohmann::json::parse(call_text_);
ToolCall c;
Expand All @@ -223,6 +267,102 @@ class AnswerStream {
mode_ = Mode::content;
}

// `call:NAME{key:value,...}` -> a ToolCall whose arguments are JSON, so the two routes hand the caller one shape.
// Throws on a body that is not that shape (the caller puts the markup back into content).
// The Gemma object syntax is not JSON: keys are bare, string values are wrapped in <|"|>, and nested objects and
// arrays use the same bare-key form. This rewrites it rather than guessing: <|"|>v<|"|> becomes a JSON string,
// a bare key gains quotes, and numbers/true/false/null pass through as literals.
// shortcut: values that are neither quoted, numeric, boolean, null nor a brace/bracket nest are emitted as strings,
// re-derive from the pack if a writer starts emitting bare tokens that are meant to be something else.
static std::string gemma_object_to_json(const std::string& body) {
std::string out;
size_t i = 0;
auto space = [&] { while (i < body.size() && (body[i] == ' ' || body[i] == '\n' || body[i] == '\t')) ++i; };
// one value at `i`
std::function<void()> value = [&] {
space();
if (body.compare(i, 5, "<|\"|>") == 0) {
i += 5;
std::string lit;
while (i < body.size() && body.compare(i, 5, "<|\"|>") != 0) lit += body[i++];
// A string whose closing escape never arrives is malformed, not a shorter string: refusing it sends
// the markup back to content (close_call), where accepting it would invent a truncated argument.
if (i >= body.size()) throw std::runtime_error("gemma4 call: unterminated <|\"|> string");
i += 5;
out += nlohmann::json(lit).dump();
return;
}
if (i < body.size() && (body[i] == '{' || body[i] == '[')) {
const char close_ch = body[i] == '{' ? '}' : ']';
out += body[i++];
space();
bool first = true;
while (i < body.size() && body[i] != close_ch) {
if (!first) out += ',';
first = false;
space();
if (close_ch == '}') {
const size_t colon = body.find(':', i);
if (colon == std::string::npos) throw std::runtime_error("gemma4 call: a key with no value");
std::string key = body.substr(i, colon - i);
while (!key.empty() && (key.back() == ' ' || key.back() == '\n')) key.pop_back();
if (key.empty()) throw std::runtime_error("gemma4 call: an empty key");
out += nlohmann::json(key).dump() + ":";
i = colon + 1;
value();
} else {
value();
}
space();
if (i < body.size() && body[i] == ',') ++i;
}
if (i >= body.size()) throw std::runtime_error("gemma4 call: an object or array that never closes");
out += body[i++];
return;
}
std::string raw;
int depth = 0;
while (i < body.size()) {
const char c = body[i];
if (c == '{' || c == '[') ++depth;
if (c == '}' || c == ']') { if (depth == 0) break; --depth; }
if (depth == 0 && (c == ',' || c == '}' || c == ']')) break;
raw += c; ++i;
}
while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\n')) raw.pop_back();
// An empty value consumed nothing. Refusing it is also what keeps the loops above finite: `[1}` would
// otherwise ask for a value at the `}` for ever, appending a comma each time.
if (raw.empty()) throw std::runtime_error("gemma4 call: an empty value");
if (raw == "true" || raw == "false" || raw == "null" || (!raw.empty() && (std::isdigit((unsigned char)raw[0]) || raw[0] == '-'))) {
out += raw;
} else {
out += nlohmann::json(raw).dump();
}
};
value();
space();
if (i != body.size()) throw std::runtime_error("gemma4 call: text after the arguments object");
return out;
}

bool close_gemma_call() {
const std::string body = answer_detail::trim(call_text_);
if (body.compare(0, 5, "call:") != 0) return false;
const size_t brace = body.find('{');
if (brace == std::string::npos || body.back() != '}') return false;
ToolCall c;
c.id = "call_" + std::to_string(calls_.size());
c.name = answer_detail::trim(body.substr(5, brace - 5));
if (c.name.empty()) return false;
try {
c.arguments = gemma_object_to_json(body.substr(brace, body.size() - brace));
} catch (const std::exception&) {
return false;
}
calls_.push_back(std::move(c));
return true;
}

AnswerFamily fam_;
answer_detail::FamilyTags tags_;
bool tools_sent_ = false;
Expand Down
47 changes: 47 additions & 0 deletions app/serve_degenerate.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A reply that is one repeated character after the context limit (engine#1779, F10 fault 2).
//
// FastFlowLM 1.0.12's notes list "Qwen3.5 answering `//////` after hitting the context limit". The rule here is deliberately
// blunt, because the thing it detects is blunt: a reply whose whole body is one character repeated. Four is the floor --
// `//////` (the reported symptom) is six, and an earlier detector in this tree used `\1{7,}`, i.e. eight, so a server
// answering exactly the reported string passed it (engine#1791, closed; the rule below is pi-hrx-sched's, credited).
//
// It is a predicate, not a policy: callers decide what to do (answer with a refusal, close the stream cleanly), and nothing
// here reads the request, the model or the context. That keeps it testable without a device or a pack -- see
// tests/serve_degenerate_test.cpp.
#ifndef ONEBIT_SERVE_DEGENERATE_H
#define ONEBIT_SERVE_DEGENERATE_H

#include <cstddef>
#include <string>

namespace onebit {

// True when `core` (the reply text with leading/trailing whitespace removed) is at least four characters and every one of
// them is the same character. An empty or short reply is not degenerate -- "ok" is a legitimate answer, and "\n\n\n\n" is
// trimmed to nothing before it gets here.
inline bool degenerate_reply(const std::string& core) {
if (core.size() < 4) return false;
for (std::size_t i = 1; i < core.size(); ++i) {
if (core[i] != core[0]) return false;
}
return true;
}

} // namespace onebit

#endif // ONEBIT_SERVE_DEGENERATE_H
Loading
Loading