From d7ad572204ad677071cd26af529de92e37ba6751 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 22:14:34 -0400 Subject: [PATCH] forward-serve: an answer that starts inside a reasoning block the prompt opened is split as reasoning (Refs #1786) Qwen3.5's template ends its generation prompt with , so the model never writes the opening tag. The splitter only entered reasoning on a tag it saw, so the whole reasoning was returned as content and the closing tag dropped. npu::prompt_opens_reasoning() tells AnswerStream, parse_answer and content_mask to start inside the block. Co-Authored-By: Claude Fable 5.2 --- app/forward_serve.cpp | 9 ++++-- app/npu_answer_parser.h | 32 ++++++++++++++++---- docs/npu-chat-template.md | 9 ++++++ tests/npu_answer_parser_test.cpp | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 0ead20865..1b4249bec 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -625,6 +625,9 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // markup whatever the container is (app/npu_tool_grammar.h), so that reply is split as qwen3. Every // other reply is in the container's own markup. const npu::AnswerFamily family = opts.tool_grammar ? npu::AnswerFamily::qwen3 : e.answer_family; + // A template whose generation prompt already opens the reasoning block leaves the answer inside it + // (npu::prompt_opens_reasoning). Only a rendered chat prompt can: an ids prompt and /v1/completions are raw. + const bool starts_in_reasoning = chat && !prompt_is_ids && npu::prompt_opens_reasoning(prompt, family); auto wrap = [&](const std::string& content, const std::string& finish, const std::vector& lps) { json out; out["id"] = id; @@ -635,7 +638,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // (issue #652 item 3) one shared parser splits what the model wrote into OpenAI's three // parts; a reply that is only tool calls says content: null. const npu::ParsedAnswer ans = - npu::parse_answer(content, family, opts.tools_sent); + npu::parse_answer(content, family, opts.tools_sent, starts_in_reasoning); json message{{"role", "assistant"}}; message["content"] = ans.content_null ? json(nullptr) : json(ans.content); if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning; @@ -656,7 +659,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib std::vector texts; texts.reserve(lps.size()); for (const auto& x : lps) texts.push_back(x.text); - const std::vector keep = npu::content_mask(texts, family, opts.tools_sent); + const std::vector keep = npu::content_mask(texts, family, opts.tools_sent, starts_in_reasoning); std::vector content_lps; for (size_t i = 0; i < lps.size(); i++) if (i < keep.size() && keep[i]) content_lps.push_back(lps[i]); @@ -703,7 +706,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib }; // The same splitter as the non-streamed path, fed piece by piece, so a streamed reply // and a non-streamed one cannot disagree about what is reasoning and what is content. - npu::AnswerStream stream(family, opts.tools_sent); + npu::AnswerStream stream(family, opts.tools_sent, starts_in_reasoning); auto emit_pieces = [&](const std::string& t, const std::vector& lp) { std::vector> pieces; stream.feed(t, pieces); diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 305a6a2c1..ab1c95ee2 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -142,8 +142,11 @@ inline AnswerFamily answer_family_of_template(const std::string& template_text) // (is_reasoning, text) pieces; finish() flushes what the holdback still held and reports the calls. class AnswerStream { public: - AnswerStream(AnswerFamily family, bool tools_sent) - : fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent) {} + // `starts_in_reasoning`: the PROMPT already opened the reasoning block (prompt_opens_reasoning below), so the + // answer begins inside it and the first thing to look for is the closing tag. + AnswerStream(AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false) + : fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent), + mode_(starts_in_reasoning ? Mode::reasoning : Mode::content) {} void feed(const std::string& chunk, std::vector>& out) { buf_ += chunk; @@ -376,12 +379,28 @@ class AnswerStream { bool saw_text_ = false; }; +// Does the rendered prompt leave the model INSIDE a reasoning block? Some templates end their generation prompt +// with the opening tag itself -- Qwen3.5's ends `<|im_start|>assistant\n\n` when thinking is on -- so the +// model never writes that tag: its answer starts with the reasoning and the first tag in it is the closing one. +// A splitter that waits to see the block opened then files the whole reasoning under `content` and drops the +// closing tag it never saw opened, so nothing in the reply marks where the answer starts (measured on FastFlowLM's +// Qwen3.5-2B-NPU2, engine#1786). True when the prompt, less trailing whitespace, +// ends with the family's opening tag. A prompt that opens and closes the block (thinking off: +// `\n\n\n\n`) does not end with it, and neither does one that never mentions it. +inline bool prompt_opens_reasoning(const std::string& prompt, AnswerFamily family) { + const std::string open = answer_detail::tags_of(family).think_open; + size_t end = prompt.size(); + while (end > 0 && (prompt[end - 1] == ' ' || prompt[end - 1] == '\n' || prompt[end - 1] == '\r' || prompt[end - 1] == '\t')) --end; + return !open.empty() && end >= open.size() && prompt.compare(end - open.size(), open.size(), open) == 0; +} + // Which of `pieces` (one generated token's text each, in order) belong to `content` -- not to // reasoning and not to call markup. Callers that report a per-token value use this so reasoning and // call markup get no entry (issue #651 item 3). A token that straddles a boundary ("The") // counts as content: its text did reach the caller. -inline std::vector content_mask(const std::vector& pieces, AnswerFamily family, bool tools_sent) { - AnswerStream s(family, tools_sent); +inline std::vector content_mask(const std::vector& pieces, AnswerFamily family, bool tools_sent, + bool starts_in_reasoning = false) { + AnswerStream s(family, tools_sent, starts_in_reasoning); std::vector mask; mask.reserve(pieces.size()); std::vector> out; @@ -406,8 +425,9 @@ inline std::vector content_mask(const std::vector& pieces, An } // The whole answer at once: the same splitter, so the two paths cannot disagree. -inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent) { - AnswerStream s(family, tools_sent); +inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent, + bool starts_in_reasoning = false) { + AnswerStream s(family, tools_sent, starts_in_reasoning); std::vector> pieces; s.feed(text, pieces); s.finish(pieces); diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index 6baf7dac5..b8581feef 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -198,6 +198,15 @@ Both give the client the same OpenAI shape: `reasoning_content`, `content`, and is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value, an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged. +**A reasoning block the prompt opened.** Some templates end their generation prompt with the opening tag itself: +Qwen3.5's ends `<|im_start|>assistant\n\n` when thinking is on. The model then never writes that tag; its +answer starts with the reasoning and the first tag in it is the closing one. `forward-serve` checks whether the +rendered prompt, less trailing whitespace, ends with the family's opening tag (`npu::prompt_opens_reasoning`), and +if so splits the answer as starting inside the block. Before engine#1786's row on FastFlowLM's `Qwen3.5-2B-NPU2` +it did not: the whole reasoning came back in `content`, `reasoning_content` was empty, and the closing tag was +dropped, so nothing in the reply marked where the answer began. With thinking off the same template opens and +closes the block in the prompt, and the answer is content from its first token. + One exception: `tool_choice: "required"` or a named function installs a grammar that constrains the reply to the `qwen3` call markup for every container (`app/npu_tool_grammar.h`), so that reply is split as `qwen3`. A grammar in Gemma's own call syntax is not written yet. diff --git a/tests/npu_answer_parser_test.cpp b/tests/npu_answer_parser_test.cpp index f79a067dc..e66c93951 100644 --- a/tests/npu_answer_parser_test.cpp +++ b/tests/npu_answer_parser_test.cpp @@ -129,6 +129,58 @@ int main() { "the qwen3 family's reasoning tags are the ones this test feeds"); } + // A reasoning block the PROMPT opened (engine#1786, FastFlowLM's Qwen3.5-2B-NPU2): the template's generation prompt + // ends with ``, so the answer starts inside the block and the only tag in it is the closing one. + { + const AnswerFamily Q = AnswerFamily::qwen3; + const std::string on = "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n\n"; + const std::string off = "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + const std::string plain = "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n"; + check(prompt_opens_reasoning(on, Q), "a prompt that ends with leaves the model inside the block"); + check(!prompt_opens_reasoning(off, Q), "one that opens AND closes it (thinking off) does not"); + check(!prompt_opens_reasoning(plain, Q) && !prompt_opens_reasoning("", Q) && !prompt_opens_reasoning("\n \n", Q), + "nor does one that never mentions it, or an empty one"); + check(!prompt_opens_reasoning("tell me what means<|im_end|>\n<|im_start|>assistant\n", Q), + "the tag earlier in the prompt is not the prompt's end"); + check(!prompt_opens_reasoning(on, AnswerFamily::gemma4), "the tag is the family's own: opens nothing for gemma4"); + + const std::string reply = "Thinking Process:\n1. The capital of France is Paris.\n\n\nParis"; + const auto before = parse_answer(reply, Q, false); // what the route did: the fault, as an assertion + check(before.reasoning.empty() && contains(before.content, "Thinking Process") && !contains(before.content, "") && + contains(before.content, "Paris"), + "without the flag the reasoning is filed under content, with the closing tag dropped, so nothing marks where the answer starts (the measured fault)"); + const auto a = parse_answer(reply, Q, false, /*starts_in_reasoning=*/true); + check(a.reasoning == "Thinking Process:\n1. The capital of France is Paris.\n", "with it the reasoning is the reasoning"); + check(a.content == "\n\nParis" && !contains(a.content, "think"), "and the content is the answer, with no tag in it"); + + // an answer cut off inside the block: all reasoning, nothing invented as content + const auto cut = parse_answer("Thinking Process:\n1. Analyze", Q, false, true); + check(cut.reasoning == "Thinking Process:\n1. Analyze" && cut.content.empty(), "an answer cut inside the block is all reasoning"); + // a call after the prompt-opened block still parses + const auto call = parse_answer("need the tool{\"name\":\"ping\",\"arguments\":{}}", Q, true, true); + check(call.reasoning == "need the tool" && call.calls.size() == 1 && call.calls[0].name == "ping" && call.content_null, + "a tool call after the prompt-opened block is still a call"); + // the model re-opening the block itself is harmless + const auto twice = parse_answer("againok", Q, false, true); + check(twice.content == "ok" && contains(twice.reasoning, "again"), "a model that writes the opening tag anyway still ends up split"); + + // streamed, split at awkward places, the pieces add up to the same thing + AnswerStream st(Q, false, true); + std::vector> out; + for (const std::string& piece : {std::string("Thinking Pro"), std::string("cess:\n1. The capital of France is Paris.\n\n\nPar"), std::string("is")}) + st.feed(piece, out); + st.finish(out); + check(st.reasoning() == a.reasoning && st.content() == a.content, "the streamed split equals the whole-answer split"); + check(!out.empty() && out.front().first, "and its first piece is reasoning, so a client sees it as such from the first token"); + + // per-token mask: reasoning tokens get no logprob entry, content tokens do + const std::vector toks = {"Thinking", " Process", "", "\n\n", "Paris"}; + const std::vector m = content_mask(toks, Q, false, true); + check(m == std::vector({0, 0, 0, 1, 1}), "the content mask starts after the closing tag"); + check(content_mask(toks, Q, false) == std::vector({1, 1, 0, 1, 1}), "where without the flag the reasoning tokens counted as content"); + } + std::printf("%s\n", failures ? "FAILED" : "all cases ok"); return failures ? 1 : 0; }