Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res
std::fclose(f);
}
}
const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr);
const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size());
// -1: the grammar allows no token here, so the answer ends where it is. An end token is
// in the mask only where the grammar can stop (refresh_allowed), so a constrained reply
// that completes reports "stop" instead of padding to the budget (engine#1182).
Expand Down Expand Up @@ -865,6 +865,10 @@ int run_forward_ids(const std::string& model_dir, const std::string& kernels, co
if (ids.empty()) throw std::runtime_error("--ids: no token ids");
const std::string model = model_dir + "/model.q4nx";
if (!std::filesystem::exists(model)) throw std::runtime_error(model + " does not exist");
// The tokenizer's size bounds the printed ids (engine#2017); a container without tokenizer.json keeps the whole row.
int tok_vocab = 0;
if (const std::string tj = model_dir + "/tokenizer.json"; std::filesystem::exists(tj))
tok_vocab = npu::Tokenizer(tj).size();
// Same device discipline as forward-serve: host-GEMM runs take no lock (engine#1353 -- a host-GEMM run that held it
// for 4 h 52 m stopped two lanes' device work), a run that opens the device takes the exclusive one.
int lock_fd = -1;
Expand Down Expand Up @@ -895,10 +899,8 @@ int run_forward_ids(const std::string& model_dir, const std::string& kernels, co
" floats for " + std::to_string(m) + " rows)");
const int nv = int(all.size() / size_t(m));
for (int r = 0; r < m; ++r) {
const float* row = all.data() + size_t(r) * size_t(nv);
int best = 0;
for (int v = 1; v < nv; ++v)
if (row[v] > row[best]) best = v;
// Bounded by the tokenizer like the served route (engine#2017): an id past it is a padding position.
const int best = npu::argmax_token(all.data() + size_t(r) * size_t(nv), size_t(nv), tok_vocab);
std::printf("%d\n", best); // token ids out, in row order: the id the model chooses at that row
}
done += m;
Expand Down
29 changes: 25 additions & 4 deletions app/npu_sampling.h
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,25 @@ struct Candidate {
int id = 0;
};

// How many leading ids of a logits row are tokens: the tokenizer's vocabulary when it is the
// smaller of the two. The head's width is the container's `vocab_size`, which may exceed the
// tokenizer's (Qwen3-0.6B: 151,936 for 151,669 ids); the ids in between are padding positions with
// no text, and a pick that lands there emits nothing and never ends the reply (engine#1182,
// #2017). `vocab` <= 0 means "no tokenizer to bound by" and keeps the whole row.
inline size_t token_bound(size_t width, int vocab) {
return (vocab > 0 && size_t(vocab) < width) ? size_t(vocab) : width;
}

// Draw one token from `logits`. `allowed`, when given, is a per-token mask (non-zero =
// the grammar accepts the token); a mask with nothing set is refused with -1 and the
// caller reports it rather than decoding an unconstrained token.
// caller reports it rather than decoding an unconstrained token. `vocab` is the
// tokenizer's size (token_bound): ids at or past it are never drawn, with or without a mask.
inline int sample_token(const std::vector<float>& logits, const SampleParams& p, std::mt19937_64& rng,
const std::vector<char>* allowed = nullptr) {
const std::vector<char>* allowed = nullptr, int vocab = 0) {
const size_t n = token_bound(logits.size(), vocab);
std::vector<Candidate> c;
c.reserve(logits.size());
for (size_t i = 0; i < logits.size(); i++) {
c.reserve(n);
for (size_t i = 0; i < n; i++) {
if (allowed && !(*allowed)[i]) continue;
c.push_back({logits[i], int(i)});
}
Expand Down Expand Up @@ -157,6 +168,16 @@ inline int sample_token(const std::vector<float>& logits, const SampleParams& p,
return c.back().id;
}

// The greedy id of one logits row of `width` floats, bounded like sample_token(): a higher
// logit wins and a tie keeps the lower id. `1bit npu-run` prints this per row.
inline int argmax_token(const float* row, size_t width, int vocab = 0) {
const size_t n = token_bound(width, vocab);
int best = 0;
for (size_t v = 1; v < n; v++)
if (row[v] > row[size_t(best)]) best = int(v);
return best;
}

// One candidate of the raw distribution, as the client reads it: the id and its log-probability.
struct Logprob {
int id = 0;
Expand Down
8 changes: 8 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -1841,6 +1841,14 @@ ended itself once its grammar had reached an accepting state — the route puts
mask exactly there, so a complete JSON object reports `stop` instead of padding with whitespace to `max_tokens`
(engine#1182).

Every pick is bounded by the tokenizer, not only the constrained one. The head is `vocab_size` wide, and
`vocab_size` may exceed the tokenizer's ids: Qwen3's 151,936 against 151,669 (measured on Qwen3-0.6B's
`config.json` and `tokenizer.json`). The ids in between are padding positions with no text, so a token drawn
there emits nothing and never ends the reply. `sample_token()` and `1bit npu-run`'s per-row argmax
(`argmax_token()`) therefore consider only ids below the tokenizer's size (`token_bound()` in
`app/npu_sampling.h`, engine#2017), with or without a grammar. A container with no `tokenizer.json` keeps the
whole row.

The sampler is llama.cpp's, without llama.cpp: `app/npu_sampling.h` repeats the semantics of
`third_party/llama.cpp/src/llama-sampler.cpp` (top_k, then top_p, then temperature, then the
cumulative-sum draw that `common/sampling.cpp`'s chain ends with) over the raw logits this route
Expand Down
31 changes: 31 additions & 0 deletions tests/npu_sampling_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ using onebit::npu::logprobs_of;
using onebit::npu::options_from_body;
using onebit::npu::SampleParams;
using onebit::npu::sample_token;
using onebit::npu::argmax_token;
using onebit::npu::token_bound;
using onebit::npu::stop_holdback;

int failures = 0;
Expand Down Expand Up @@ -1192,7 +1194,36 @@ void test_request_parsing() {

} // namespace

// engine#2017: the logits row is `vocab_size` wide, which may exceed the tokenizer (Qwen3-0.6B: 151,936 for 151,669
// ids; the shape below is the issue's, 32,032 for 32,016). Ids past the tokenizer are padding positions: every real
// logit negative and the padding at exactly 0.0 is the row where an unbounded argmax picks one (engine#1182).
void test_vocab_bound() {
const int V = 32016, W = 32032;
std::vector<float> row(size_t(W), 0.0f);
for (int v = 0; v < V; v++) row[size_t(v)] = -1.0f - float(v % 97) * 0.01f;
row[123] = -0.5f; // the real best
check(argmax_token(row.data(), size_t(W)) >= V, "control: an unbounded argmax lands on a padding id");
check(argmax_token(row.data(), size_t(W), V) == 123, "the bounded argmax picks the best real id");
check(sample(row, SampleParams(), 1) >= V, "control: unbounded greedy sample lands on a padding id");
std::mt19937_64 rng(1);
check(sample_token(row, SampleParams(), rng, nullptr, V) == 123, "bounded greedy sample picks the best real id");
SampleParams hot;
hot.temperature = 1.0f;
bool all_real = true;
for (uint64_t s = 0; s < 200; s++) {
std::mt19937_64 r(s);
if (sample_token(row, hot, r, nullptr, V) >= V) all_real = false;
}
check(all_real, "a bounded draw at temperature 1 never draws a padding id");
std::vector<char> mask(size_t(W), 1);
std::mt19937_64 r2(2);
check(sample_token(row, SampleParams(), r2, &mask, V) == 123, "the bound holds with a mask that allows padding");
check(token_bound(size_t(W), 0) == size_t(W), "no tokenizer keeps the whole row");
check(token_bound(size_t(V), W) == size_t(V), "a tokenizer larger than the row does not widen it");
}

int main() {
test_vocab_bound();
test_sampler();
test_sampler_distribution();
test_stop();
Expand Down
Loading