Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -352,18 +352,27 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
// allowed[i] says token i can be appended without leaving the grammar. The first
// byte test rejects almost every token cheaply (a JSON object at the start accepts
// a handful of first bytes out of 256), so only the survivors pay a full feed().
// The ids a constrained answer may end on: the container's own eos ids plus <|endoftext|>.
std::vector<int> end_ids = e.eos;
if (e.endoftext >= 0) end_ids.push_back(e.endoftext);
// Ids at or beyond the tokenizer's vocabulary are padding positions, not tokens: decode()
// returns "" for them, which used to make them indistinguishable from a byte fragment and so
// unconditionally allowed (engine#1182: the model drifts into them once the grammar leaves it
// nothing to say, and they emit no text and never end the reply).
const size_t vocab = size_t(e.tok->size());
auto refresh_allowed = [&]() {
allowed.assign(logits.size(), 0);
const npu::ByteMask first = gram->next_bytes(grammar_state);
for (size_t i = 0; i < logits.size(); i++) {
for (size_t i = 0; i < logits.size() && i < vocab; i++) {
const std::string& t = e.tok_text[i];
if (t.empty()) {
allowed[i] = 1; // an empty piece cannot leave the grammar
allowed[i] = 1; // a half-character byte fragment cannot leave the grammar
} else if (first.has(static_cast<unsigned char>(t[0]))) {
npu::Grammar::State probe = grammar_state;
allowed[i] = gram->feed(probe, t) ? 1 : 0;
}
}
npu::allow_end_tokens(allowed, grammar_state, end_ids);
};

std::string pending; // decoded, not yet sent: a stop sequence may still match in it
Expand Down Expand Up @@ -403,9 +412,9 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
}
}
const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr);
// -1: the grammar allows no token here, so the answer ends where it is. EOS is
// not masked by the grammar on purpose: a constrained answer is still allowed to
// stop early (the client validates it), which beats a runaway inside the mask.
// -1: the grammar allows no token here, so the answer ends where it is. An end token is
// in the mask only where the grammar can stop (refresh_allowed), so a constrained reply
// that completes reports "stop" instead of padding to the budget (engine#1182).
if (next < 0) { stopped_at_eos = true; break; }
if (std::find(e.eos.begin(), e.eos.end(), next) != e.eos.end() ||
(e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; break; }
Expand Down
14 changes: 14 additions & 0 deletions app/npu_grammar.h
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,20 @@ inline uint64_t gbnf_frame(uint32_t rule, uint32_t pos) { return (uint64_t(rule)
inline uint32_t gbnf_rule(uint64_t f) { return uint32_t(f >> 32); }
inline uint32_t gbnf_pos(uint64_t f) { return uint32_t(f & 0xffffffffu); }

// Let a constrained answer stop where the grammar can stop. A mask built only from tokens whose
// text the grammar accepts has no way to end a reply: the model's end token (`<|im_end|>` on Qwen,
// `<|endoftext|>`) has that literal text, which is not in the constrained language, so EOS is masked
// out at every position and the decode can only run to max_tokens (engine#1182: a complete JSON
// object, then 13 tokens of whitespace, finish_reason "length"). Setting the end tokens' bits once
// the parse is at an accepting state makes stopping possible; before that they stay masked, so a
// constrained answer can never be cut into something the grammar would not accept.
inline void allow_end_tokens(std::vector<char>& allowed, const Grammar::State& st,
const std::vector<int>& ends) {
if (!Grammar::can_stop(st)) return;
for (const int id : ends)
if (id >= 0 && size_t(id) < allowed.size()) allowed[size_t(id)] = 1;
}

// Recursive-descent parser for the GBNF text, emitting into Grammar's rule vectors.
// Repetition uses llama.cpp's rewrite rules (third_party/llama.cpp/src/llama-grammar.cpp,
// handle_repetitions): S{m,n} becomes m copies plus n-m generated tail rules, S* becomes
Expand Down
5 changes: 4 additions & 1 deletion docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1005,7 +1005,10 @@ supported by this route's schema-to-GBNF converter", "`gbnf: unterminated litera
"`stream`: must be true or false", "`tools`: must be an array"),
never a silently ignored key and never a 500: a wrong *type* is a client error like any other, including on the
fields the route reads before it starts decoding (engine#1106). `finish_reason` is `length` when `max_tokens` ran out, and `stop` for
an end-of-sequence token, a stop sequence or a grammar that allows nothing more.
an end-of-sequence token, a stop sequence, a grammar that allows nothing more, or a constrained answer the model
ended itself once its grammar had reached an accepting state — the route puts the container's end tokens into the
mask exactly there, so a complete JSON object reports `stop` instead of padding with whitespace to `max_tokens`
(engine#1182).

The sampler is llama.cpp's, without llama.cpp: `app/npu_sampling.h` repeats the semantics of
`third_party/llama.cpp/src/llama-sampler.cpp` (top_k, then top_p, then temperature, then the
Expand Down
44 changes: 44 additions & 0 deletions tests/npu_sampling_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
namespace {

using onebit::npu::find_stop;
using onebit::npu::allow_end_tokens;
using onebit::npu::Grammar;
using onebit::npu::AnswerFamily;
using onebit::npu::AnswerStream;
Expand Down Expand Up @@ -281,6 +282,48 @@ void test_json_grammar() {
check(st.stacks.size() > 0, "the refused token left the state where it was");
}

// engine#1182: a mask built only from grammar-legal tokens drops the end token, so a constrained
// answer could never stop and padded to max_tokens with finish_reason "length".
void test_grammar_end_tokens() {
const std::vector<int> ends = {151645, 151643, -1}; // Qwen's <|im_end|>, <|endoftext|>, and a miss
{
// `root ::= "ab"`: mid-grammar there is no stopping point, so every end token stays masked.
const Grammar g(R"(root ::= "ab")");
Grammar::State st = g.start();
std::vector<char> allowed(200000, 0);
allow_end_tokens(allowed, st, ends);
check(allowed[151645] == 0 && allowed[151643] == 0,
"EOS is masked before the grammar can end, so a constrained answer cannot be truncated");
check(g.feed(st, "ab"), "the grammar's own text is accepted");
allow_end_tokens(allowed, st, ends);
check(allowed[151645] == 1 && allowed[151643] == 1,
"EOS is offered once the grammar is at an accepting state");
check(allowed[0] == 0 && allowed[151644] == 0, "only the end tokens are added, nothing else");
}
{
// The real case: the house JSON grammar after a complete object, where the reply must be
// able to stop even though more properties would still be legal.
const Grammar g(json_grammar());
Grammar::State st = g.start();
check(g.feed(st, R"({"a": 1})"), "a complete object is accepted");
check(Grammar::can_stop(st), "a complete object is an accepting state");
std::vector<char> allowed(200000, 0);
allow_end_tokens(allowed, st, ends);
check(allowed[151645] == 1, "json_schema/json_object replies may stop on EOS when the JSON is whole");
}
{
// An id outside the mask (a vocabulary smaller than the end ids) is ignored, not written.
const Grammar g(R"(root ::= "ab")");
Grammar::State st = g.start();
g.feed(st, "ab");
std::vector<char> small(8, 0);
allow_end_tokens(small, st, ends);
int set = 0;
for (char c : small) set += c ? 1 : 0;
check(set == 0, "an end token outside the mask vector is ignored rather than written out of bounds");
}
}

void test_gbnf() {
{
const Grammar g(R"(root ::= "yes" | "no")");
Expand Down Expand Up @@ -927,6 +970,7 @@ int main() {
test_stop();
test_json_grammar();
test_gbnf();
test_grammar_end_tokens();
test_logprobs();
test_logprob_options();
test_prefix_cache();
Expand Down
Loading