Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,13 @@ endif()
add_executable(think_split_test tests/think_split_test.cpp)
target_include_directories(think_split_test PRIVATE app)
add_test(NAME think_split COMMAND think_split_test)
# engine#2135: what enable_thinking can mean for a pack, read off its own chat template (app/think_template.h).
add_executable(think_template_test tests/think_template_test.cpp)
target_include_directories(think_template_test PRIVATE app)
add_test(NAME think_template COMMAND think_template_test)
# ci.yml's build list is not ours to edit without `workflow` scope, so the test reaches CI as a build dependency of a
# target it already builds (tools/ci/ci_targets_check.py follows this edge).
add_dependencies(onebit think_template_test)
# Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI.
# engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device.
add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp)
Expand Down
99 changes: 99 additions & 0 deletions app/think_template.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// think_template.h -- what `chat_template_kwargs.enable_thinking` can mean for a pack, read off its own template
// (engine#2135).
//
// The unified NPU route builds ChatML itself instead of rendering the pack's template, and for a thinking-off request
// it appends an empty block, `<think>\n\n</think>\n\n`, which is what Qwen3's own template emits. That is right only for
// a template that HAS the switch. Two packs on this box do not:
//
// Qwen3-4B-Thinking-2507 its generation prompt opens `<think>\n` unconditionally and `enable_thinking` does not occur.
// Sent the empty block it reasons anyway, in plain text, and closes with `</think>`; with the
// splitter off for a thinking-off request, all of it came back as `content`.
// Qwen3-VL-4B-Instruct neither `enable_thinking` nor `<think>` occurs. Sent the empty block it answered a turn
// that already held a block it has never seen: "I apologize, but I cannot provide that
// information ..." to "What is the capital of France?".
//
// So the prompt follows the template: a switch -> unchanged; no switch and the template opens the block -> thinking
// cannot be turned off, the request is served as thinking-on; no switch and no block -> the request adds nothing.
// Header-only and pure, so the rule is host-tested (tests/think_template_test.cpp).
#pragma once

#include <string>

namespace onebit {

// What a chat template says about thinking. `known` is false when no template was read (the rule then changes nothing).
struct ThinkTemplate {
bool known = false;
bool has_switch = false; // the template reads `enable_thinking`
bool mentions_think = false; // `<think>` occurs anywhere in it
bool opens_think = false; // its generation prompt itself opens `<think>`
};

inline ThinkTemplate think_template_of(const std::string& t) {
ThinkTemplate tt;
if (t.empty()) return tt;
tt.known = true;
tt.has_switch = t.find("enable_thinking") != std::string::npos;
tt.mentions_think = t.find("<think>") != std::string::npos;
// The generation prompt is what the last `add_generation_prompt` block emits, up to its `endif`.
const size_t g = t.rfind("add_generation_prompt");
if (g != std::string::npos) {
const size_t end = t.find("endif", g);
const std::string block = t.substr(g, end == std::string::npos ? std::string::npos : end - g);
// Opening means leaving the block open: Qwen3's own generation block also holds `<think>`, but only inside the
// empty `<think>\n\n</think>` it emits for thinking off, which closes again.
tt.opens_think = block.find("<think>") != std::string::npos && block.find("</think>") == std::string::npos;
}
return tt;
}

// What a chat request's prompt ends with after `<|im_start|>assistant\n`.
enum class ThinkPrompt { none, open, empty_block };

struct ThinkPlan {
ThinkPrompt prompt = ThinkPrompt::none;
bool start_inside = false; // the splitter starts inside the block (the prompt opened it)
bool split = false; // reasoning_content is split out of the reply
};

// `want_thinking`: the request's enable_thinking (default true). `route_opens_think`: what the route says its prompt
// does when thinking is on. `reasoning_none`: the request asked for reasoning_format "none".
inline ThinkPlan think_plan(bool chat, bool want_thinking, bool reasoning_none, bool route_opens_think,
const ThinkTemplate& tt) {
ThinkPlan p;
if (!chat) return p;
bool thinking = want_thinking;
bool add_empty_block = !want_thinking;
if (!want_thinking && tt.known && !tt.has_switch) {
if (tt.opens_think) {
thinking = true; // the template cannot turn thinking off: serve what it would send
add_empty_block = false;
} else if (!tt.mentions_think) {
add_empty_block = false; // a template with no block at all: thinking off adds nothing
}
}
if (add_empty_block) p.prompt = ThinkPrompt::empty_block;
else if (thinking && route_opens_think) p.prompt = ThinkPrompt::open;
p.start_inside = p.prompt == ThinkPrompt::open;
// Split whenever the caller did not ask for raw text: a model that reasons regardless is still split, and a reply
// with no block passes through the splitter unchanged (the way llama-server's reasoning_format "auto" works).
p.split = !reasoning_none;
return p;
}

} // namespace onebit
62 changes: 52 additions & 10 deletions app/unified.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "unified.h"
#include "npu_queue.h"
#include "think_split.h"
#include "think_template.h" // engine#2135: what enable_thinking can mean, read off the pack's template

#include "generate.h"
#include "model.h"
Expand All @@ -47,6 +48,7 @@
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <memory>
#include <mutex>
#include <string>
Expand Down Expand Up @@ -74,6 +76,7 @@ struct Engine {
int max_context = npu::Lane::kMaxContext;
float default_penalty = 1.1f; // greedy 0.6B models loop without it
bool open_think = false; // the template opens "<think>\n" when thinking is on
ThinkTemplate think_tmpl; // engine#2135: the pack's own chat_template, as far as thinking goes
std::timed_mutex mu; // timed: the wait for the device is bounded (engine#911)
// How long a STREAMED request may wait for the device before it is told the device is busy (engine#1024).
// The one-device lock below serialises generations, and an *unbounded, silent* wait is what made a
Expand Down Expand Up @@ -117,9 +120,39 @@ struct Engine {
// The chat scaffold for the model families the lane serves. Hugging Face ships
// the template as Jinja; ChatML is what Qwen2 and Qwen3 render it to for plain
// text messages, with generation starting after "<|im_start|>assistant\n".
// open_think: the template opens the think block itself when thinking is on (Qwen3.6's
// does; Qwen3's leaves it to the model).
std::string chatml(const json& messages, bool thinking = true, bool open_think = false) {
// The pack's chat template, or "" when it has none this can read: tokenizer_config.json's `chat_template` (a string,
// or a list of {name, template} where "default" wins), else chat_template.jinja. Only think_template_of() reads it;
// "" makes that rule change nothing (engine#2135).
std::string read_chat_template(const std::string& model_dir) {
try {
std::ifstream f(model_dir + "/tokenizer_config.json");
if (f) {
const json cfg = json::parse(f, nullptr, /*allow_exceptions=*/false);
if (cfg.is_object() && cfg.contains("chat_template")) {
const json& t = cfg["chat_template"];
if (t.is_string()) return t.get<std::string>();
if (t.is_array()) {
std::string first;
for (const auto& one : t) {
if (!one.is_object() || !one.contains("template") || !one["template"].is_string()) continue;
if (one.value("name", "") == "default") return one["template"].get<std::string>();
if (first.empty()) first = one["template"].get<std::string>();
}
if (!first.empty()) return first;
}
}
}
std::ifstream j(model_dir + "/chat_template.jinja");
if (j) return std::string((std::istreambuf_iterator<char>(j)), std::istreambuf_iterator<char>());
} catch (const std::exception&) {
}
return {};
}

// `think`: what follows the assistant prefix (think_template.h decides it from the request and the pack's template):
// an opened block (Qwen3.6's template opens one when thinking is on), the empty block Qwen3's template emits for
// enable_thinking false, or nothing.
std::string chatml(const json& messages, ThinkPrompt think = ThinkPrompt::none) {
std::string p;
for (const auto& m : messages) {
std::string content;
Expand All @@ -133,10 +166,10 @@ std::string chatml(const json& messages, bool thinking = true, bool open_think =
p += "<|im_start|>" + m.at("role").get<std::string>() + "\n" + content + "<|im_end|>\n";
}
p += "<|im_start|>assistant\n";
// chat_template_kwargs.enable_thinking = false: what Qwen3's own template
// emits, an empty think block, so the model answers directly.
if (!thinking) p += "<think>\n\n</think>\n\n";
else if (open_think) p += "<think>\n";
// chat_template_kwargs.enable_thinking = false on a template with the switch: what Qwen3's own template emits, an
// empty think block, so the model answers directly. On a template without it, think_plan() never asks for this.
if (think == ThinkPrompt::empty_block) p += "<think>\n\n</think>\n\n";
else if (think == ThinkPrompt::open) p += "<think>\n";
return p;
}

Expand Down Expand Up @@ -174,9 +207,12 @@ void handle_generate(Engine& e, const httplib::Request& req, httplib::Response&
bool thinking = true;
if (body.contains("chat_template_kwargs") && body["chat_template_kwargs"].is_object())
thinking = body["chat_template_kwargs"].value("enable_thinking", true);
prompt = chat ? chatml(body.at("messages"), thinking, e.open_think) : body.at("prompt").get<std::string>();
split_reasoning = chat && thinking && body.value("reasoning_format", std::string("auto")) != "none";
think_opened = thinking && e.open_think;
// engine#2135: the prompt and the split follow the pack's template, not only the request's flag.
const ThinkPlan plan = think_plan(chat, thinking, body.value("reasoning_format", std::string("auto")) == "none",
e.open_think, e.think_tmpl);
prompt = chat ? chatml(body.at("messages"), plan.prompt) : body.at("prompt").get<std::string>();
split_reasoning = chat && plan.split;
think_opened = plan.start_inside;
} catch (const std::exception& ex) {
res.status = 400;
res.set_content(json{{"error", {{"message", std::string("bad request: ") + ex.what()}}}}.dump(), "application/json");
Expand Down Expand Up @@ -534,6 +570,12 @@ int run_unified(int argc, char** argv) {
e.max_context = e.priv->max_context();
e.default_penalty = e.priv->default_repetition_penalty();
e.open_think = e.priv->opens_think();
e.think_tmpl = think_template_of(read_chat_template(model_dir));
if (e.think_tmpl.known && !e.think_tmpl.has_switch)
std::fprintf(stderr, "1bit unified: the chat template has no enable_thinking switch%s; enable_thinking false %s\n",
e.think_tmpl.opens_think ? " and opens <think> itself" : "",
e.think_tmpl.opens_think ? "is served as thinking on" :
e.think_tmpl.mentions_think ? "keeps the empty block" : "adds nothing to the prompt");
std::fprintf(stderr, "1bit unified: on the private route %s%s%s\n", route.name.c_str(),
e.priv->describe().empty() ? "" : ": ", e.priv->describe().c_str());
} else {
Expand Down
8 changes: 8 additions & 0 deletions docs/serve.md
Original file line number Diff line number Diff line change
Expand Up @@ -534,6 +534,14 @@ backends apply the model's own chat template. The NPU route emits what Qwen3's
template does: an empty think block after the assistant prefix. For Qwen3.6-35B-A3B it
also opens `<think>\n` when thinking is on, as that model's template does.

On the fused NPU route that empty block is only sent when the pack's own `chat_template` has the
switch (engine#2135). The route reads the template at load and says what it decided:
a template that opens `<think>` itself and has no switch (FastFlowLM's
`Qwen3-4B-Thinking-2507`) cannot turn thinking off, so the request is served as thinking on and
the reasoning comes back as `reasoning_content`; a template with neither a switch nor a think
block (`Qwen3-VL-4B-Instruct`) gets nothing added. The reply is split into `reasoning_content`
whenever `reasoning_format` is not `none`, whatever `enable_thinking` says.

For a `.gguf` the engine starts that server as a private child on a loopback
port and forwards the OpenAI routes to it, streaming included. Replies carry
the served model name. `auto` means HRX for GGUF: it met RFC #213's decode gates (docs/hrx.md), and
Expand Down
99 changes: 99 additions & 0 deletions tests/think_template_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// think_template_test: what enable_thinking can mean for a pack, read off its template (app/think_template.h,
// engine#2135). The three generation blocks below are copied from the packs on this box: Qwen3-0.6B (has the switch),
// FastFlowLM's Qwen3-4B-Thinking-2507-NPU2 (opens <think> unconditionally, no switch) and Qwen3-VL-4B-Instruct-NPU2
// (neither).
#include "think_template.h"

#include <cstdio>
#include <string>

static int fails = 0;
static void check(bool ok, const char* what) {
std::printf("%s %s\n", ok ? "ok " : "FAIL", what);
if (!ok) ++fails;
}

using onebit::ThinkPlan;
using onebit::ThinkPrompt;
using onebit::think_plan;
using onebit::think_template_of;

int main() {
const std::string qwen3 =
"{%- for message in messages %}...<think>\\n' + reasoning_content ...{%- endfor %}\n"
"{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n"
" {%- if enable_thinking is defined and enable_thinking is false %}\n"
" {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}";
const std::string thinking_only =
"{%- for message in messages %}... '</think>' ...{%- endfor %}\n"
"{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n<think>\\n' }}\n{%- endif %}";
const std::string no_think =
"{%- for message in messages %}...{%- endfor %}\n"
"{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n";

const auto q = think_template_of(qwen3), t = think_template_of(thinking_only), v = think_template_of(no_think);
check(q.known && q.has_switch && q.mentions_think && !q.opens_think,
"Qwen3: a switch; its empty <think></think> for thinking off is not 'opening the block'");
check(t.known && !t.has_switch && t.mentions_think && t.opens_think,
"Thinking-2507: no switch, and the generation prompt leaves <think> open");
check(v.known && !v.has_switch && !v.mentions_think && !v.opens_think, "VL-4B-Instruct: no switch and no block");
check(!think_template_of("").known, "no template read: not known");

// enable_thinking false
auto plan = [](const onebit::ThinkTemplate& tt, bool want, bool route_opens = false, bool none = false) {
return think_plan(true, want, none, route_opens, tt);
};
ThinkPlan p = plan(q, false);
check(p.prompt == ThinkPrompt::empty_block && !p.start_inside,
"Qwen3, thinking off: the empty block, as before (the 0.6B demo path does not move)");
p = plan(t, false);
check(p.prompt == ThinkPrompt::none && !p.start_inside && p.split,
"Thinking-2507, thinking off: no empty block -- served as thinking on, and split");
p = plan(t, false, /*route_opens=*/true);
check(p.prompt == ThinkPrompt::open && p.start_inside && p.split,
"... and where the route opens the block for thinking on, the splitter starts inside it");
p = plan(v, false);
check(p.prompt == ThinkPrompt::none && !p.start_inside, "VL-4B-Instruct, thinking off: nothing is added to the prompt");
p = plan(onebit::ThinkTemplate{}, false);
check(p.prompt == ThinkPrompt::empty_block, "an unknown template keeps today's empty block");
// a template that mentions <think> somewhere but has no switch and does not open it: today's empty block
const auto mention = think_template_of("... <think> in a message ...{%- if add_generation_prompt %}{{- 'assistant\\n' }}{%- endif %}");
check(plan(mention, false).prompt == ThinkPrompt::empty_block, "no switch, a <think> only in the history: unchanged");

// a template WITH the switch that opens the block only when thinking is on (Qwen3.6's shape): thinking off must
// still mean thinking off -- the switch wins over "opens the block"
const auto q36 = think_template_of(
"{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}"
"{%- if enable_thinking is not defined or enable_thinking %}{{- '<think>\\n' }}{%- endif %}{%- endif %}");
check(q36.has_switch && q36.opens_think, "Qwen3.6 shape: a switch, and the block opened when thinking is on");
check(plan(q36, false).prompt == ThinkPrompt::empty_block, "... so thinking off is still the empty block, not forced on");

// enable_thinking true never moves
for (const auto* tt : {&q, &t, &v}) {
check(plan(*tt, true).prompt == ThinkPrompt::none && plan(*tt, true, true).prompt == ThinkPrompt::open,
"thinking on: the route decides, whatever the template");
}
// the splitter: on unless reasoning_format is "none"; off for a raw completion
check(plan(q, false).split && plan(q, true).split, "the splitter is on with thinking on or off");
check(!plan(t, false, false, /*none=*/true).split, "reasoning_format none: no split");
check(!think_plan(false, false, false, false, t).split && think_plan(false, false, false, false, t).prompt == ThinkPrompt::none,
"not a chat request: nothing");

std::printf("%s\n", fails ? "think_template: FAILED" : "think_template: all cases ok");
return fails ? 1 : 0;
}
Loading