From 33320af6bd19481e0a2f159f05bc16d72e41882a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 18:59:56 -0400 Subject: [PATCH] npu: a grammar-constrained reply can stop, so json_schema/json_object stop padding to max_tokens (#1182) A mask built only out of tokens whose text the grammar accepts has no way to end a reply. The end token is a token like any other and its text is "<|im_end|>" (or "<|endoftext|>"), which is not in the constrained language, so the mask removed it at every position; sample_token() drops every id whose bit is 0, so EOS was unreachable rather than merely unlikely, and the decode loop could only ever end at max_tokens. The comment at the EOS check said the opposite of what the code did. Measured on Qwen3-0.6B, host GEMM, max_tokens 32, before -> after: json_schema 32 tokens, complete object + 13 whitespace tokens, "length" -> 14 tokens, "stop" json_object 32 tokens, "{}" + the same whitespace tail, "length" -> 1 token, "stop" client GBNF 32 tokens, "length" -> 4 tokens, "stop" max_tokens 8 (too small for the answer) -> still "length", i.e. truncation is still reported as truncation no grammar unchanged npu::allow_end_tokens() sets the request's end tokens' bits only where Grammar::can_stop() says the parse may end, so mid-grammar EOS stays masked and a constrained answer can never be cut into something the grammar would not accept; at an accepting state the reply ends with finish_reason "stop". Ids at or beyond the tokenizer's vocabulary are now masked out of the constrained set: they are padding positions, not tokens, decode() returns "" for them, and the "an empty piece cannot leave the grammar" rule was admitting them unconditionally -- the model drifted into them once the grammar left it nothing to say (the trace shows ids 151672/151680 against size() == 151669). --- app/forward_serve.cpp | 19 +++++++++++----- app/npu_grammar.h | 14 ++++++++++++ docs/npu.md | 5 ++++- tests/npu_sampling_test.cpp | 44 +++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index eecfddb5f..84c5c15f8 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -352,18 +352,27 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // allowed[i] says token i can be appended without leaving the grammar. The first // byte test rejects almost every token cheaply (a JSON object at the start accepts // a handful of first bytes out of 256), so only the survivors pay a full feed(). + // The ids a constrained answer may end on: the container's own eos ids plus <|endoftext|>. + std::vector end_ids = e.eos; + if (e.endoftext >= 0) end_ids.push_back(e.endoftext); + // Ids at or beyond the tokenizer's vocabulary are padding positions, not tokens: decode() + // returns "" for them, which used to make them indistinguishable from a byte fragment and so + // unconditionally allowed (engine#1182: the model drifts into them once the grammar leaves it + // nothing to say, and they emit no text and never end the reply). + const size_t vocab = size_t(e.tok->size()); auto refresh_allowed = [&]() { allowed.assign(logits.size(), 0); const npu::ByteMask first = gram->next_bytes(grammar_state); - for (size_t i = 0; i < logits.size(); i++) { + for (size_t i = 0; i < logits.size() && i < vocab; i++) { const std::string& t = e.tok_text[i]; if (t.empty()) { - allowed[i] = 1; // an empty piece cannot leave the grammar + allowed[i] = 1; // a half-character byte fragment cannot leave the grammar } else if (first.has(static_cast(t[0]))) { npu::Grammar::State probe = grammar_state; allowed[i] = gram->feed(probe, t) ? 1 : 0; } } + npu::allow_end_tokens(allowed, grammar_state, end_ids); }; std::string pending; // decoded, not yet sent: a stop sequence may still match in it @@ -403,9 +412,9 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res } } const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr); - // -1: the grammar allows no token here, so the answer ends where it is. EOS is - // not masked by the grammar on purpose: a constrained answer is still allowed to - // stop early (the client validates it), which beats a runaway inside the mask. + // -1: the grammar allows no token here, so the answer ends where it is. An end token is + // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply + // that completes reports "stop" instead of padding to the budget (engine#1182). if (next < 0) { stopped_at_eos = true; break; } if (std::find(e.eos.begin(), e.eos.end(), next) != e.eos.end() || (e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; break; } diff --git a/app/npu_grammar.h b/app/npu_grammar.h index 393b57e3f..388e7925f 100644 --- a/app/npu_grammar.h +++ b/app/npu_grammar.h @@ -114,6 +114,20 @@ inline uint64_t gbnf_frame(uint32_t rule, uint32_t pos) { return (uint64_t(rule) inline uint32_t gbnf_rule(uint64_t f) { return uint32_t(f >> 32); } inline uint32_t gbnf_pos(uint64_t f) { return uint32_t(f & 0xffffffffu); } +// Let a constrained answer stop where the grammar can stop. A mask built only from tokens whose +// text the grammar accepts has no way to end a reply: the model's end token (`<|im_end|>` on Qwen, +// `<|endoftext|>`) has that literal text, which is not in the constrained language, so EOS is masked +// out at every position and the decode can only run to max_tokens (engine#1182: a complete JSON +// object, then 13 tokens of whitespace, finish_reason "length"). Setting the end tokens' bits once +// the parse is at an accepting state makes stopping possible; before that they stay masked, so a +// constrained answer can never be cut into something the grammar would not accept. +inline void allow_end_tokens(std::vector& allowed, const Grammar::State& st, + const std::vector& ends) { + if (!Grammar::can_stop(st)) return; + for (const int id : ends) + if (id >= 0 && size_t(id) < allowed.size()) allowed[size_t(id)] = 1; +} + // Recursive-descent parser for the GBNF text, emitting into Grammar's rule vectors. // Repetition uses llama.cpp's rewrite rules (third_party/llama.cpp/src/llama-grammar.cpp, // handle_repetitions): S{m,n} becomes m copies plus n-m generated tail rules, S* becomes diff --git a/docs/npu.md b/docs/npu.md index 3bb2ce1e0..831c6116b 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1005,7 +1005,10 @@ supported by this route's schema-to-GBNF converter", "`gbnf: unterminated litera "`stream`: must be true or false", "`tools`: must be an array"), never a silently ignored key and never a 500: a wrong *type* is a client error like any other, including on the fields the route reads before it starts decoding (engine#1106). `finish_reason` is `length` when `max_tokens` ran out, and `stop` for -an end-of-sequence token, a stop sequence or a grammar that allows nothing more. +an end-of-sequence token, a stop sequence, a grammar that allows nothing more, or a constrained answer the model +ended itself once its grammar had reached an accepting state — the route puts the container's end tokens into the +mask exactly there, so a complete JSON object reports `stop` instead of padding with whitespace to `max_tokens` +(engine#1182). The sampler is llama.cpp's, without llama.cpp: `app/npu_sampling.h` repeats the semantics of `third_party/llama.cpp/src/llama-sampler.cpp` (top_k, then top_p, then temperature, then the diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index dc92aee13..28bc24fac 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -31,6 +31,7 @@ namespace { using onebit::npu::find_stop; +using onebit::npu::allow_end_tokens; using onebit::npu::Grammar; using onebit::npu::AnswerFamily; using onebit::npu::AnswerStream; @@ -281,6 +282,48 @@ void test_json_grammar() { check(st.stacks.size() > 0, "the refused token left the state where it was"); } +// engine#1182: a mask built only from grammar-legal tokens drops the end token, so a constrained +// answer could never stop and padded to max_tokens with finish_reason "length". +void test_grammar_end_tokens() { + const std::vector ends = {151645, 151643, -1}; // Qwen's <|im_end|>, <|endoftext|>, and a miss + { + // `root ::= "ab"`: mid-grammar there is no stopping point, so every end token stays masked. + const Grammar g(R"(root ::= "ab")"); + Grammar::State st = g.start(); + std::vector allowed(200000, 0); + allow_end_tokens(allowed, st, ends); + check(allowed[151645] == 0 && allowed[151643] == 0, + "EOS is masked before the grammar can end, so a constrained answer cannot be truncated"); + check(g.feed(st, "ab"), "the grammar's own text is accepted"); + allow_end_tokens(allowed, st, ends); + check(allowed[151645] == 1 && allowed[151643] == 1, + "EOS is offered once the grammar is at an accepting state"); + check(allowed[0] == 0 && allowed[151644] == 0, "only the end tokens are added, nothing else"); + } + { + // The real case: the house JSON grammar after a complete object, where the reply must be + // able to stop even though more properties would still be legal. + const Grammar g(json_grammar()); + Grammar::State st = g.start(); + check(g.feed(st, R"({"a": 1})"), "a complete object is accepted"); + check(Grammar::can_stop(st), "a complete object is an accepting state"); + std::vector allowed(200000, 0); + allow_end_tokens(allowed, st, ends); + check(allowed[151645] == 1, "json_schema/json_object replies may stop on EOS when the JSON is whole"); + } + { + // An id outside the mask (a vocabulary smaller than the end ids) is ignored, not written. + const Grammar g(R"(root ::= "ab")"); + Grammar::State st = g.start(); + g.feed(st, "ab"); + std::vector small(8, 0); + allow_end_tokens(small, st, ends); + int set = 0; + for (char c : small) set += c ? 1 : 0; + check(set == 0, "an end token outside the mask vector is ignored rather than written out of bounds"); + } +} + void test_gbnf() { { const Grammar g(R"(root ::= "yes" | "no")"); @@ -927,6 +970,7 @@ int main() { test_stop(); test_json_grammar(); test_gbnf(); + test_grammar_end_tokens(); test_logprobs(); test_logprob_options(); test_prefix_cache();