Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,9 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// markup whatever the container is (app/npu_tool_grammar.h), so that reply is split as qwen3. Every
// other reply is in the container's own markup.
const npu::AnswerFamily family = opts.tool_grammar ? npu::AnswerFamily::qwen3 : e.answer_family;
// A template whose generation prompt already opens the reasoning block leaves the answer inside it
// (npu::prompt_opens_reasoning). Only a rendered chat prompt can: an ids prompt and /v1/completions are raw.
const bool starts_in_reasoning = chat && !prompt_is_ids && npu::prompt_opens_reasoning(prompt, family);
auto wrap = [&](const std::string& content, const std::string& finish, const std::vector<Emitted>& lps) {
json out;
out["id"] = id;
Expand All @@ -635,7 +638,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// (issue #652 item 3) one shared parser splits what the model wrote into OpenAI's three
// parts; a reply that is only tool calls says content: null.
const npu::ParsedAnswer ans =
npu::parse_answer(content, family, opts.tools_sent);
npu::parse_answer(content, family, opts.tools_sent, starts_in_reasoning);
json message{{"role", "assistant"}};
message["content"] = ans.content_null ? json(nullptr) : json(ans.content);
if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning;
Expand All @@ -656,7 +659,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
std::vector<std::string> texts;
texts.reserve(lps.size());
for (const auto& x : lps) texts.push_back(x.text);
const std::vector<char> keep = npu::content_mask(texts, family, opts.tools_sent);
const std::vector<char> keep = npu::content_mask(texts, family, opts.tools_sent, starts_in_reasoning);
std::vector<Emitted> content_lps;
for (size_t i = 0; i < lps.size(); i++)
if (i < keep.size() && keep[i]) content_lps.push_back(lps[i]);
Expand Down Expand Up @@ -703,7 +706,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
};
// The same splitter as the non-streamed path, fed piece by piece, so a streamed reply
// and a non-streamed one cannot disagree about what is reasoning and what is content.
npu::AnswerStream stream(family, opts.tools_sent);
npu::AnswerStream stream(family, opts.tools_sent, starts_in_reasoning);
auto emit_pieces = [&](const std::string& t, const std::vector<Emitted>& lp) {
std::vector<std::pair<bool, std::string>> pieces;
stream.feed(t, pieces);
Expand Down
32 changes: 26 additions & 6 deletions app/npu_answer_parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,11 @@ inline AnswerFamily answer_family_of_template(const std::string& template_text)
// (is_reasoning, text) pieces; finish() flushes what the holdback still held and reports the calls.
class AnswerStream {
public:
AnswerStream(AnswerFamily family, bool tools_sent)
: fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent) {}
// `starts_in_reasoning`: the PROMPT already opened the reasoning block (prompt_opens_reasoning below), so the
// answer begins inside it and the first thing to look for is the closing tag.
AnswerStream(AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false)
: fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent),
mode_(starts_in_reasoning ? Mode::reasoning : Mode::content) {}

void feed(const std::string& chunk, std::vector<std::pair<bool, std::string>>& out) {
buf_ += chunk;
Expand Down Expand Up @@ -376,12 +379,28 @@ class AnswerStream {
bool saw_text_ = false;
};

// Does the rendered prompt leave the model INSIDE a reasoning block? Some templates end their generation prompt
// with the opening tag itself -- Qwen3.5's ends `<|im_start|>assistant\n<think>\n` when thinking is on -- so the
// model never writes that tag: its answer starts with the reasoning and the first tag in it is the closing one.
// A splitter that waits to see the block opened then files the whole reasoning under `content` and drops the
// closing tag it never saw opened, so nothing in the reply marks where the answer starts (measured on FastFlowLM's
// Qwen3.5-2B-NPU2, engine#1786). True when the prompt, less trailing whitespace,
// ends with the family's opening tag. A prompt that opens and closes the block (thinking off:
// `<think>\n\n</think>\n\n`) does not end with it, and neither does one that never mentions it.
inline bool prompt_opens_reasoning(const std::string& prompt, AnswerFamily family) {
const std::string open = answer_detail::tags_of(family).think_open;
size_t end = prompt.size();
while (end > 0 && (prompt[end - 1] == ' ' || prompt[end - 1] == '\n' || prompt[end - 1] == '\r' || prompt[end - 1] == '\t')) --end;
return !open.empty() && end >= open.size() && prompt.compare(end - open.size(), open.size(), open) == 0;
}

// Which of `pieces` (one generated token's text each, in order) belong to `content` -- not to
// reasoning and not to call markup. Callers that report a per-token value use this so reasoning and
// call markup get no entry (issue #651 item 3). A token that straddles a boundary ("</think>The")
// counts as content: its text did reach the caller.
inline std::vector<char> content_mask(const std::vector<std::string>& pieces, AnswerFamily family, bool tools_sent) {
AnswerStream s(family, tools_sent);
inline std::vector<char> content_mask(const std::vector<std::string>& pieces, AnswerFamily family, bool tools_sent,
bool starts_in_reasoning = false) {
AnswerStream s(family, tools_sent, starts_in_reasoning);
std::vector<char> mask;
mask.reserve(pieces.size());
std::vector<std::pair<bool, std::string>> out;
Expand All @@ -406,8 +425,9 @@ inline std::vector<char> content_mask(const std::vector<std::string>& pieces, An
}

// The whole answer at once: the same splitter, so the two paths cannot disagree.
inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent) {
AnswerStream s(family, tools_sent);
inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent,
bool starts_in_reasoning = false) {
AnswerStream s(family, tools_sent, starts_in_reasoning);
std::vector<std::pair<bool, std::string>> pieces;
s.feed(text, pieces);
s.finish(pieces);
Expand Down
9 changes: 9 additions & 0 deletions docs/npu-chat-template.md
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,15 @@ Both give the client the same OpenAI shape: `reasoning_content`, `content`, and
is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value,
an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged.

**A reasoning block the prompt opened.** Some templates end their generation prompt with the opening tag itself:
Qwen3.5's ends `<|im_start|>assistant\n<think>\n` when thinking is on. The model then never writes that tag; its
answer starts with the reasoning and the first tag in it is the closing one. `forward-serve` checks whether the
rendered prompt, less trailing whitespace, ends with the family's opening tag (`npu::prompt_opens_reasoning`), and
if so splits the answer as starting inside the block. Before engine#1786's row on FastFlowLM's `Qwen3.5-2B-NPU2`
it did not: the whole reasoning came back in `content`, `reasoning_content` was empty, and the closing tag was
dropped, so nothing in the reply marked where the answer began. With thinking off the same template opens and
closes the block in the prompt, and the answer is content from its first token.

One exception: `tool_choice: "required"` or a named function installs a grammar that constrains the reply to
the `qwen3` call markup for every container (`app/npu_tool_grammar.h`), so that reply is split as `qwen3`.
A grammar in Gemma's own call syntax is not written yet.
Expand Down
52 changes: 52 additions & 0 deletions tests/npu_answer_parser_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,58 @@ int main() {
"the qwen3 family's reasoning tags are the ones this test feeds");
}

// A reasoning block the PROMPT opened (engine#1786, FastFlowLM's Qwen3.5-2B-NPU2): the template's generation prompt
// ends with `<think>`, so the answer starts inside the block and the only tag in it is the closing one.
{
const AnswerFamily Q = AnswerFamily::qwen3;
const std::string on = "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n<think>\n";
const std::string off = "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n";
const std::string plain = "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n";
check(prompt_opens_reasoning(on, Q), "a prompt that ends with <think> leaves the model inside the block");
check(!prompt_opens_reasoning(off, Q), "one that opens AND closes it (thinking off) does not");
check(!prompt_opens_reasoning(plain, Q) && !prompt_opens_reasoning("", Q) && !prompt_opens_reasoning("\n \n", Q),
"nor does one that never mentions it, or an empty one");
check(!prompt_opens_reasoning("tell me what <think> means<|im_end|>\n<|im_start|>assistant\n", Q),
"the tag earlier in the prompt is not the prompt's end");
check(!prompt_opens_reasoning(on, AnswerFamily::gemma4), "the tag is the family's own: <think> opens nothing for gemma4");

const std::string reply = "Thinking Process:\n1. The capital of France is Paris.\n</think>\n\nParis";
const auto before = parse_answer(reply, Q, false); // what the route did: the fault, as an assertion
check(before.reasoning.empty() && contains(before.content, "Thinking Process") && !contains(before.content, "</think>") &&
contains(before.content, "Paris"),
"without the flag the reasoning is filed under content, with the closing tag dropped, so nothing marks where the answer starts (the measured fault)");
const auto a = parse_answer(reply, Q, false, /*starts_in_reasoning=*/true);
check(a.reasoning == "Thinking Process:\n1. The capital of France is Paris.\n", "with it the reasoning is the reasoning");
check(a.content == "\n\nParis" && !contains(a.content, "think"), "and the content is the answer, with no tag in it");

// an answer cut off inside the block: all reasoning, nothing invented as content
const auto cut = parse_answer("Thinking Process:\n1. Analyze", Q, false, true);
check(cut.reasoning == "Thinking Process:\n1. Analyze" && cut.content.empty(), "an answer cut inside the block is all reasoning");
// a call after the prompt-opened block still parses
const auto call = parse_answer("need the tool</think><tool_call>{\"name\":\"ping\",\"arguments\":{}}</tool_call>", Q, true, true);
check(call.reasoning == "need the tool" && call.calls.size() == 1 && call.calls[0].name == "ping" && call.content_null,
"a tool call after the prompt-opened block is still a call");
// the model re-opening the block itself is harmless
const auto twice = parse_answer("<think>again</think>ok", Q, false, true);
check(twice.content == "ok" && contains(twice.reasoning, "again"), "a model that writes the opening tag anyway still ends up split");

// streamed, split at awkward places, the pieces add up to the same thing
AnswerStream st(Q, false, true);
std::vector<std::pair<bool, std::string>> out;
for (const std::string& piece : {std::string("Thinking Pro"), std::string("cess:\n1. The capital of France is Paris.\n</th"),
std::string("ink>\n\nPar"), std::string("is")})
st.feed(piece, out);
st.finish(out);
check(st.reasoning() == a.reasoning && st.content() == a.content, "the streamed split equals the whole-answer split");
check(!out.empty() && out.front().first, "and its first piece is reasoning, so a client sees it as such from the first token");

// per-token mask: reasoning tokens get no logprob entry, content tokens do
const std::vector<std::string> toks = {"Thinking", " Process", "</think>", "\n\n", "Paris"};
const std::vector<char> m = content_mask(toks, Q, false, true);
check(m == std::vector<char>({0, 0, 0, 1, 1}), "the content mask starts after the closing tag");
check(content_mask(toks, Q, false) == std::vector<char>({1, 1, 0, 1, 1}), "where without the flag the reasoning tokens counted as content");
}

std::printf("%s\n", failures ? "FAILED" : "all cases ok");
return failures ? 1 : 0;
}
Loading