diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e1a46c35..d6823978d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -465,6 +465,13 @@ endif() add_executable(think_split_test tests/think_split_test.cpp) target_include_directories(think_split_test PRIVATE app) add_test(NAME think_split COMMAND think_split_test) +# engine#2135: what enable_thinking can mean for a pack, read off its own chat template (app/think_template.h). +add_executable(think_template_test tests/think_template_test.cpp) +target_include_directories(think_template_test PRIVATE app) +add_test(NAME think_template COMMAND think_template_test) +# ci.yml's build list is not ours to edit without `workflow` scope, so the test reaches CI as a build dependency of a +# target it already builds (tools/ci/ci_targets_check.py follows this edge). +add_dependencies(onebit think_template_test) # Host / Origin checks every engine HTTP server applies (app/http_guard.h); header only, runs in CI. # engine#1779 (F10 fault 2): the degenerate-repetition predicate at the context limit. Header only, no device. add_executable(serve_degenerate_test tests/serve_degenerate_test.cpp) diff --git a/app/think_template.h b/app/think_template.h new file mode 100644 index 000000000..fecf9e070 --- /dev/null +++ b/app/think_template.h @@ -0,0 +1,99 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// think_template.h -- what `chat_template_kwargs.enable_thinking` can mean for a pack, read off its own template +// (engine#2135). +// +// The unified NPU route builds ChatML itself instead of rendering the pack's template, and for a thinking-off request +// it appends an empty block, `\n\n\n\n`, which is what Qwen3's own template emits. That is right only for +// a template that HAS the switch. Two packs on this box do not: +// +// Qwen3-4B-Thinking-2507 its generation prompt opens `\n` unconditionally and `enable_thinking` does not occur. +// Sent the empty block it reasons anyway, in plain text, and closes with ``; with the +// splitter off for a thinking-off request, all of it came back as `content`. +// Qwen3-VL-4B-Instruct neither `enable_thinking` nor `` occurs. Sent the empty block it answered a turn +// that already held a block it has never seen: "I apologize, but I cannot provide that +// information ..." to "What is the capital of France?". +// +// So the prompt follows the template: a switch -> unchanged; no switch and the template opens the block -> thinking +// cannot be turned off, the request is served as thinking-on; no switch and no block -> the request adds nothing. +// Header-only and pure, so the rule is host-tested (tests/think_template_test.cpp). +#pragma once + +#include + +namespace onebit { + +// What a chat template says about thinking. `known` is false when no template was read (the rule then changes nothing). +struct ThinkTemplate { + bool known = false; + bool has_switch = false; // the template reads `enable_thinking` + bool mentions_think = false; // `` occurs anywhere in it + bool opens_think = false; // its generation prompt itself opens `` +}; + +inline ThinkTemplate think_template_of(const std::string& t) { + ThinkTemplate tt; + if (t.empty()) return tt; + tt.known = true; + tt.has_switch = t.find("enable_thinking") != std::string::npos; + tt.mentions_think = t.find("") != std::string::npos; + // The generation prompt is what the last `add_generation_prompt` block emits, up to its `endif`. + const size_t g = t.rfind("add_generation_prompt"); + if (g != std::string::npos) { + const size_t end = t.find("endif", g); + const std::string block = t.substr(g, end == std::string::npos ? std::string::npos : end - g); + // Opening means leaving the block open: Qwen3's own generation block also holds ``, but only inside the + // empty `\n\n` it emits for thinking off, which closes again. + tt.opens_think = block.find("") != std::string::npos && block.find("") == std::string::npos; + } + return tt; +} + +// What a chat request's prompt ends with after `<|im_start|>assistant\n`. +enum class ThinkPrompt { none, open, empty_block }; + +struct ThinkPlan { + ThinkPrompt prompt = ThinkPrompt::none; + bool start_inside = false; // the splitter starts inside the block (the prompt opened it) + bool split = false; // reasoning_content is split out of the reply +}; + +// `want_thinking`: the request's enable_thinking (default true). `route_opens_think`: what the route says its prompt +// does when thinking is on. `reasoning_none`: the request asked for reasoning_format "none". +inline ThinkPlan think_plan(bool chat, bool want_thinking, bool reasoning_none, bool route_opens_think, + const ThinkTemplate& tt) { + ThinkPlan p; + if (!chat) return p; + bool thinking = want_thinking; + bool add_empty_block = !want_thinking; + if (!want_thinking && tt.known && !tt.has_switch) { + if (tt.opens_think) { + thinking = true; // the template cannot turn thinking off: serve what it would send + add_empty_block = false; + } else if (!tt.mentions_think) { + add_empty_block = false; // a template with no block at all: thinking off adds nothing + } + } + if (add_empty_block) p.prompt = ThinkPrompt::empty_block; + else if (thinking && route_opens_think) p.prompt = ThinkPrompt::open; + p.start_inside = p.prompt == ThinkPrompt::open; + // Split whenever the caller did not ask for raw text: a model that reasons regardless is still split, and a reply + // with no block passes through the splitter unchanged (the way llama-server's reasoning_format "auto" works). + p.split = !reasoning_none; + return p; +} + +} // namespace onebit diff --git a/app/unified.cpp b/app/unified.cpp index c44f2a436..446d6dacf 100644 --- a/app/unified.cpp +++ b/app/unified.cpp @@ -28,6 +28,7 @@ #include "unified.h" #include "npu_queue.h" #include "think_split.h" +#include "think_template.h" // engine#2135: what enable_thinking can mean, read off the pack's template #include "generate.h" #include "model.h" @@ -47,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +76,7 @@ struct Engine { int max_context = npu::Lane::kMaxContext; float default_penalty = 1.1f; // greedy 0.6B models loop without it bool open_think = false; // the template opens "\n" when thinking is on + ThinkTemplate think_tmpl; // engine#2135: the pack's own chat_template, as far as thinking goes std::timed_mutex mu; // timed: the wait for the device is bounded (engine#911) // How long a STREAMED request may wait for the device before it is told the device is busy (engine#1024). // The one-device lock below serialises generations, and an *unbounded, silent* wait is what made a @@ -117,9 +120,39 @@ struct Engine { // The chat scaffold for the model families the lane serves. Hugging Face ships // the template as Jinja; ChatML is what Qwen2 and Qwen3 render it to for plain // text messages, with generation starting after "<|im_start|>assistant\n". -// open_think: the template opens the think block itself when thinking is on (Qwen3.6's -// does; Qwen3's leaves it to the model). -std::string chatml(const json& messages, bool thinking = true, bool open_think = false) { +// The pack's chat template, or "" when it has none this can read: tokenizer_config.json's `chat_template` (a string, +// or a list of {name, template} where "default" wins), else chat_template.jinja. Only think_template_of() reads it; +// "" makes that rule change nothing (engine#2135). +std::string read_chat_template(const std::string& model_dir) { + try { + std::ifstream f(model_dir + "/tokenizer_config.json"); + if (f) { + const json cfg = json::parse(f, nullptr, /*allow_exceptions=*/false); + if (cfg.is_object() && cfg.contains("chat_template")) { + const json& t = cfg["chat_template"]; + if (t.is_string()) return t.get(); + if (t.is_array()) { + std::string first; + for (const auto& one : t) { + if (!one.is_object() || !one.contains("template") || !one["template"].is_string()) continue; + if (one.value("name", "") == "default") return one["template"].get(); + if (first.empty()) first = one["template"].get(); + } + if (!first.empty()) return first; + } + } + } + std::ifstream j(model_dir + "/chat_template.jinja"); + if (j) return std::string((std::istreambuf_iterator(j)), std::istreambuf_iterator()); + } catch (const std::exception&) { + } + return {}; +} + +// `think`: what follows the assistant prefix (think_template.h decides it from the request and the pack's template): +// an opened block (Qwen3.6's template opens one when thinking is on), the empty block Qwen3's template emits for +// enable_thinking false, or nothing. +std::string chatml(const json& messages, ThinkPrompt think = ThinkPrompt::none) { std::string p; for (const auto& m : messages) { std::string content; @@ -133,10 +166,10 @@ std::string chatml(const json& messages, bool thinking = true, bool open_think = p += "<|im_start|>" + m.at("role").get() + "\n" + content + "<|im_end|>\n"; } p += "<|im_start|>assistant\n"; - // chat_template_kwargs.enable_thinking = false: what Qwen3's own template - // emits, an empty think block, so the model answers directly. - if (!thinking) p += "\n\n\n\n"; - else if (open_think) p += "\n"; + // chat_template_kwargs.enable_thinking = false on a template with the switch: what Qwen3's own template emits, an + // empty think block, so the model answers directly. On a template without it, think_plan() never asks for this. + if (think == ThinkPrompt::empty_block) p += "\n\n\n\n"; + else if (think == ThinkPrompt::open) p += "\n"; return p; } @@ -174,9 +207,12 @@ void handle_generate(Engine& e, const httplib::Request& req, httplib::Response& bool thinking = true; if (body.contains("chat_template_kwargs") && body["chat_template_kwargs"].is_object()) thinking = body["chat_template_kwargs"].value("enable_thinking", true); - prompt = chat ? chatml(body.at("messages"), thinking, e.open_think) : body.at("prompt").get(); - split_reasoning = chat && thinking && body.value("reasoning_format", std::string("auto")) != "none"; - think_opened = thinking && e.open_think; + // engine#2135: the prompt and the split follow the pack's template, not only the request's flag. + const ThinkPlan plan = think_plan(chat, thinking, body.value("reasoning_format", std::string("auto")) == "none", + e.open_think, e.think_tmpl); + prompt = chat ? chatml(body.at("messages"), plan.prompt) : body.at("prompt").get(); + split_reasoning = chat && plan.split; + think_opened = plan.start_inside; } catch (const std::exception& ex) { res.status = 400; res.set_content(json{{"error", {{"message", std::string("bad request: ") + ex.what()}}}}.dump(), "application/json"); @@ -534,6 +570,12 @@ int run_unified(int argc, char** argv) { e.max_context = e.priv->max_context(); e.default_penalty = e.priv->default_repetition_penalty(); e.open_think = e.priv->opens_think(); + e.think_tmpl = think_template_of(read_chat_template(model_dir)); + if (e.think_tmpl.known && !e.think_tmpl.has_switch) + std::fprintf(stderr, "1bit unified: the chat template has no enable_thinking switch%s; enable_thinking false %s\n", + e.think_tmpl.opens_think ? " and opens itself" : "", + e.think_tmpl.opens_think ? "is served as thinking on" : + e.think_tmpl.mentions_think ? "keeps the empty block" : "adds nothing to the prompt"); std::fprintf(stderr, "1bit unified: on the private route %s%s%s\n", route.name.c_str(), e.priv->describe().empty() ? "" : ": ", e.priv->describe().c_str()); } else { diff --git a/docs/serve.md b/docs/serve.md index 226cb3d72..c541f86df 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -534,6 +534,14 @@ backends apply the model's own chat template. The NPU route emits what Qwen3's template does: an empty think block after the assistant prefix. For Qwen3.6-35B-A3B it also opens `\n` when thinking is on, as that model's template does. +On the fused NPU route that empty block is only sent when the pack's own `chat_template` has the +switch (engine#2135). The route reads the template at load and says what it decided: +a template that opens `` itself and has no switch (FastFlowLM's +`Qwen3-4B-Thinking-2507`) cannot turn thinking off, so the request is served as thinking on and +the reasoning comes back as `reasoning_content`; a template with neither a switch nor a think +block (`Qwen3-VL-4B-Instruct`) gets nothing added. The reply is split into `reasoning_content` +whenever `reasoning_format` is not `none`, whatever `enable_thinking` says. + For a `.gguf` the engine starts that server as a private child on a loopback port and forwards the OpenAI routes to it, streaming included. Replies carry the served model name. `auto` means HRX for GGUF: it met RFC #213's decode gates (docs/hrx.md), and diff --git a/tests/think_template_test.cpp b/tests/think_template_test.cpp new file mode 100644 index 000000000..c7fae9e41 --- /dev/null +++ b/tests/think_template_test.cpp @@ -0,0 +1,99 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// think_template_test: what enable_thinking can mean for a pack, read off its template (app/think_template.h, +// engine#2135). The three generation blocks below are copied from the packs on this box: Qwen3-0.6B (has the switch), +// FastFlowLM's Qwen3-4B-Thinking-2507-NPU2 (opens unconditionally, no switch) and Qwen3-VL-4B-Instruct-NPU2 +// (neither). +#include "think_template.h" + +#include +#include + +static int fails = 0; +static void check(bool ok, const char* what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) ++fails; +} + +using onebit::ThinkPlan; +using onebit::ThinkPrompt; +using onebit::think_plan; +using onebit::think_template_of; + +int main() { + const std::string qwen3 = + "{%- for message in messages %}...\\n' + reasoning_content ...{%- endfor %}\n" + "{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n" + " {%- if enable_thinking is defined and enable_thinking is false %}\n" + " {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}"; + const std::string thinking_only = + "{%- for message in messages %}... '' ...{%- endfor %}\n" + "{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n\\n' }}\n{%- endif %}"; + const std::string no_think = + "{%- for message in messages %}...{%- endfor %}\n" + "{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n"; + + const auto q = think_template_of(qwen3), t = think_template_of(thinking_only), v = think_template_of(no_think); + check(q.known && q.has_switch && q.mentions_think && !q.opens_think, + "Qwen3: a switch; its empty for thinking off is not 'opening the block'"); + check(t.known && !t.has_switch && t.mentions_think && t.opens_think, + "Thinking-2507: no switch, and the generation prompt leaves open"); + check(v.known && !v.has_switch && !v.mentions_think && !v.opens_think, "VL-4B-Instruct: no switch and no block"); + check(!think_template_of("").known, "no template read: not known"); + + // enable_thinking false + auto plan = [](const onebit::ThinkTemplate& tt, bool want, bool route_opens = false, bool none = false) { + return think_plan(true, want, none, route_opens, tt); + }; + ThinkPlan p = plan(q, false); + check(p.prompt == ThinkPrompt::empty_block && !p.start_inside, + "Qwen3, thinking off: the empty block, as before (the 0.6B demo path does not move)"); + p = plan(t, false); + check(p.prompt == ThinkPrompt::none && !p.start_inside && p.split, + "Thinking-2507, thinking off: no empty block -- served as thinking on, and split"); + p = plan(t, false, /*route_opens=*/true); + check(p.prompt == ThinkPrompt::open && p.start_inside && p.split, + "... and where the route opens the block for thinking on, the splitter starts inside it"); + p = plan(v, false); + check(p.prompt == ThinkPrompt::none && !p.start_inside, "VL-4B-Instruct, thinking off: nothing is added to the prompt"); + p = plan(onebit::ThinkTemplate{}, false); + check(p.prompt == ThinkPrompt::empty_block, "an unknown template keeps today's empty block"); + // a template that mentions somewhere but has no switch and does not open it: today's empty block + const auto mention = think_template_of("... in a message ...{%- if add_generation_prompt %}{{- 'assistant\\n' }}{%- endif %}"); + check(plan(mention, false).prompt == ThinkPrompt::empty_block, "no switch, a only in the history: unchanged"); + + // a template WITH the switch that opens the block only when thinking is on (Qwen3.6's shape): thinking off must + // still mean thinking off -- the switch wins over "opens the block" + const auto q36 = think_template_of( + "{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}" + "{%- if enable_thinking is not defined or enable_thinking %}{{- '\\n' }}{%- endif %}{%- endif %}"); + check(q36.has_switch && q36.opens_think, "Qwen3.6 shape: a switch, and the block opened when thinking is on"); + check(plan(q36, false).prompt == ThinkPrompt::empty_block, "... so thinking off is still the empty block, not forced on"); + + // enable_thinking true never moves + for (const auto* tt : {&q, &t, &v}) { + check(plan(*tt, true).prompt == ThinkPrompt::none && plan(*tt, true, true).prompt == ThinkPrompt::open, + "thinking on: the route decides, whatever the template"); + } + // the splitter: on unless reasoning_format is "none"; off for a raw completion + check(plan(q, false).split && plan(q, true).split, "the splitter is on with thinking on or off"); + check(!plan(t, false, false, /*none=*/true).split, "reasoning_format none: no split"); + check(!think_plan(false, false, false, false, t).split && think_plan(false, false, false, false, t).prompt == ThinkPrompt::none, + "not a chat request: nothing"); + + std::printf("%s\n", fails ? "think_template: FAILED" : "think_template: all cases ok"); + return fails ? 1 : 0; +}