From c3efffe1f6ec5c5a4a1a45eb505f787d0f94438c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 17:56:57 -0400 Subject: [PATCH] npu: bound every token pick by the tokenizer, not only the grammar's (Fixes #2017) The logits row is the container's vocab_size wide, and vocab_size can exceed the tokenizer: Qwen3-0.6B has 151,936 against 151,669 ids. Only the grammar path masked the ids in between (#1182). The unconstrained sample_token() and npu-run's greedy loop could still pick one, so the reply emitted nothing and never ended. token_bound() caps both picks at the tokenizer's size. sample_token() takes it as `vocab`, and argmax_token() is npu-run's per-row argmax. The head and the verifier's M x NV block are unchanged. Co-authored-by: bong-water-water-bong Co-Authored-By: Claude Opus 5.5 --- app/forward_serve.cpp | 12 +++++++----- app/npu_sampling.h | 29 +++++++++++++++++++++++++---- docs/npu.md | 8 ++++++++ tests/npu_sampling_test.cpp | 31 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index effed11be..aa4d104f3 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -488,7 +488,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res std::fclose(f); } } - const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr); + const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size()); // -1: the grammar allows no token here, so the answer ends where it is. An end token is // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply // that completes reports "stop" instead of padding to the budget (engine#1182). @@ -848,6 +848,10 @@ int run_forward_ids(const std::string& model_dir, const std::string& kernels, co if (ids.empty()) throw std::runtime_error("--ids: no token ids"); const std::string model = model_dir + "/model.q4nx"; if (!std::filesystem::exists(model)) throw std::runtime_error(model + " does not exist"); + // The tokenizer's size bounds the printed ids (engine#2017); a container without tokenizer.json keeps the whole row. + int tok_vocab = 0; + if (const std::string tj = model_dir + "/tokenizer.json"; std::filesystem::exists(tj)) + tok_vocab = npu::Tokenizer(tj).size(); // Same device discipline as forward-serve: host-GEMM runs take no lock (engine#1353 -- a host-GEMM run that held it // for 4 h 52 m stopped two lanes' device work), a run that opens the device takes the exclusive one. int lock_fd = -1; @@ -878,10 +882,8 @@ int run_forward_ids(const std::string& model_dir, const std::string& kernels, co " floats for " + std::to_string(m) + " rows)"); const int nv = int(all.size() / size_t(m)); for (int r = 0; r < m; ++r) { - const float* row = all.data() + size_t(r) * size_t(nv); - int best = 0; - for (int v = 1; v < nv; ++v) - if (row[v] > row[best]) best = v; + // Bounded by the tokenizer like the served route (engine#2017): an id past it is a padding position. + const int best = npu::argmax_token(all.data() + size_t(r) * size_t(nv), size_t(nv), tok_vocab); std::printf("%d\n", best); // token ids out, in row order: the id the model chooses at that row } done += m; diff --git a/app/npu_sampling.h b/app/npu_sampling.h index 4e2f41b6f..065005cb6 100644 --- a/app/npu_sampling.h +++ b/app/npu_sampling.h @@ -82,14 +82,25 @@ struct Candidate { int id = 0; }; +// How many leading ids of a logits row are tokens: the tokenizer's vocabulary when it is the +// smaller of the two. The head's width is the container's `vocab_size`, which may exceed the +// tokenizer's (Qwen3-0.6B: 151,936 for 151,669 ids); the ids in between are padding positions with +// no text, and a pick that lands there emits nothing and never ends the reply (engine#1182, +// #2017). `vocab` <= 0 means "no tokenizer to bound by" and keeps the whole row. +inline size_t token_bound(size_t width, int vocab) { + return (vocab > 0 && size_t(vocab) < width) ? size_t(vocab) : width; +} + // Draw one token from `logits`. `allowed`, when given, is a per-token mask (non-zero = // the grammar accepts the token); a mask with nothing set is refused with -1 and the -// caller reports it rather than decoding an unconstrained token. +// caller reports it rather than decoding an unconstrained token. `vocab` is the +// tokenizer's size (token_bound): ids at or past it are never drawn, with or without a mask. inline int sample_token(const std::vector& logits, const SampleParams& p, std::mt19937_64& rng, - const std::vector* allowed = nullptr) { + const std::vector* allowed = nullptr, int vocab = 0) { + const size_t n = token_bound(logits.size(), vocab); std::vector c; - c.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); i++) { + c.reserve(n); + for (size_t i = 0; i < n; i++) { if (allowed && !(*allowed)[i]) continue; c.push_back({logits[i], int(i)}); } @@ -157,6 +168,16 @@ inline int sample_token(const std::vector& logits, const SampleParams& p, return c.back().id; } +// The greedy id of one logits row of `width` floats, bounded like sample_token(): a higher +// logit wins and a tie keeps the lower id. `1bit npu-run` prints this per row. +inline int argmax_token(const float* row, size_t width, int vocab = 0) { + const size_t n = token_bound(width, vocab); + int best = 0; + for (size_t v = 1; v < n; v++) + if (row[v] > row[size_t(best)]) best = int(v); + return best; +} + // One candidate of the raw distribution, as the client reads it: the id and its log-probability. struct Logprob { int id = 0; diff --git a/docs/npu.md b/docs/npu.md index ff48d28d6..ec7407506 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1727,6 +1727,14 @@ ended itself once its grammar had reached an accepting state — the route puts mask exactly there, so a complete JSON object reports `stop` instead of padding with whitespace to `max_tokens` (engine#1182). +Every pick is bounded by the tokenizer, not only the constrained one. The head is `vocab_size` wide, and +`vocab_size` may exceed the tokenizer's ids: Qwen3's 151,936 against 151,669 (measured on Qwen3-0.6B's +`config.json` and `tokenizer.json`). The ids in between are padding positions with no text, so a token drawn +there emits nothing and never ends the reply. `sample_token()` and `1bit npu-run`'s per-row argmax +(`argmax_token()`) therefore consider only ids below the tokenizer's size (`token_bound()` in +`app/npu_sampling.h`, engine#2017), with or without a grammar. A container with no `tokenizer.json` keeps the +whole row. + The sampler is llama.cpp's, without llama.cpp: `app/npu_sampling.h` repeats the semantics of `third_party/llama.cpp/src/llama-sampler.cpp` (top_k, then top_p, then temperature, then the cumulative-sum draw that `common/sampling.cpp`'s chain ends with) over the raw logits this route diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 56851d2b4..45932a92b 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -49,6 +49,8 @@ using onebit::npu::logprobs_of; using onebit::npu::options_from_body; using onebit::npu::SampleParams; using onebit::npu::sample_token; +using onebit::npu::argmax_token; +using onebit::npu::token_bound; using onebit::npu::stop_holdback; int failures = 0; @@ -1192,7 +1194,36 @@ void test_request_parsing() { } // namespace +// engine#2017: the logits row is `vocab_size` wide, which may exceed the tokenizer (Qwen3-0.6B: 151,936 for 151,669 +// ids; the shape below is the issue's, 32,032 for 32,016). Ids past the tokenizer are padding positions: every real +// logit negative and the padding at exactly 0.0 is the row where an unbounded argmax picks one (engine#1182). +void test_vocab_bound() { + const int V = 32016, W = 32032; + std::vector row(size_t(W), 0.0f); + for (int v = 0; v < V; v++) row[size_t(v)] = -1.0f - float(v % 97) * 0.01f; + row[123] = -0.5f; // the real best + check(argmax_token(row.data(), size_t(W)) >= V, "control: an unbounded argmax lands on a padding id"); + check(argmax_token(row.data(), size_t(W), V) == 123, "the bounded argmax picks the best real id"); + check(sample(row, SampleParams(), 1) >= V, "control: unbounded greedy sample lands on a padding id"); + std::mt19937_64 rng(1); + check(sample_token(row, SampleParams(), rng, nullptr, V) == 123, "bounded greedy sample picks the best real id"); + SampleParams hot; + hot.temperature = 1.0f; + bool all_real = true; + for (uint64_t s = 0; s < 200; s++) { + std::mt19937_64 r(s); + if (sample_token(row, hot, r, nullptr, V) >= V) all_real = false; + } + check(all_real, "a bounded draw at temperature 1 never draws a padding id"); + std::vector mask(size_t(W), 1); + std::mt19937_64 r2(2); + check(sample_token(row, SampleParams(), r2, &mask, V) == 123, "the bound holds with a mask that allows padding"); + check(token_bound(size_t(W), 0) == size_t(W), "no tokenizer keeps the whole row"); + check(token_bound(size_t(V), W) == size_t(V), "a tokenizer larger than the row does not widen it"); +} + int main() { + test_vocab_bound(); test_sampler(); test_sampler_distribution(); test_stop();