Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,14 @@ if(TARGET OpenMP::OpenMP_CXX)
target_link_libraries(mtp_draft_test PRIVATE OpenMP::OpenMP_CXX)
endif()
add_test(NAME mtp_draft COMMAND mtp_draft_test)
# engine#1781 gate 2: mtp::run_from() under a sampler (RowSampler) -- the reply distribution of both round shapes
# against the exact product of a toy chain's conditionals, plain sampling as the control, and forward-serve's own
# sampler (app/npu_spec_sampler.h) with temperature, top_k and top_p.
add_executable(mtp_sampled_test tests/mtp_sampled_test.cpp)
target_include_directories(mtp_sampled_test PRIVATE npu app)
target_link_libraries(mtp_sampled_test PRIVATE nlohmann_json::nlohmann_json)
add_dependencies(onebit mtp_sampled_test)
add_test(NAME mtp_sampled COMMAND mtp_sampled_test)
# engine#1781: the prompt-lookup drafter (npu/mtp/prompt_lookup.h), by hand and through mtp::run() on a toy base.
add_executable(prompt_lookup_test tests/prompt_lookup_test.cpp)
target_include_directories(prompt_lookup_test PRIVATE npu)
Expand Down
38 changes: 32 additions & 6 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
#include "npu_prefix_cache.h"
#include "npu_prompt_ids.h"
#include "npu_sampling.h"
#include "npu_spec_sampler.h"
#include "npu_stop_tokens.h"
#include "stream_bridge.h"
#include "npu_systemone.h"
Expand Down Expand Up @@ -146,6 +147,11 @@ struct ForwardEngine {
// drafting for the rest of a reply once its last 3 rounds averaged fewer than X tokens (0 = never).
int spec_lookup = 0;
double spec_fallback = 0.0;
// --spec-sample: a sampled request (temperature > 0) takes the speculative path too, its drafts kept or refused by
// the rule that keeps the target's distribution (npu/mtp/spec_accept.h through mtp::RowSampler). Off by default: the
// reply then follows the same distribution but not the plain loop's random stream, so a seed gives a different
// reply than it does without the flag.
bool spec_sample = false;
bool host_gemm = false; // NPU_INFER_HOST_GEMM: the forward runs on the host, with no launch width of its own
// --spec-draft <dir> (engine#1781): a DFlash draft package as the drafter instead of the prompt lookup, up to
// spec_draft_k drafts a round. The forward writes the draft's tap layers into draft_taps on every pass; the sink
Expand Down Expand Up @@ -585,8 +591,10 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// argmax the verify rows check drafts against; no grammar (its mask is per token and per state); no logprobs
// and no logit dump (both read the logits of every position, and a verify batch's rows are not kept). The text
// is then the plain loop's token for token -- what tests/serve_spec_lookup_test.py holds it to.
const bool spec =
(e.spec_lookup > 0 || e.draft) && opts.sample.greedy() && !gram && !want_lp && e.dump_logits.empty();
// --spec-sample widens it to sampled requests, whose drafts are then judged by the distribution-keeping rule
// instead (ServeSampler below): the reply follows the same distribution as the plain loop's, not its stream.
const bool spec = (e.spec_lookup > 0 || e.draft) && (opts.sample.greedy() || e.spec_sample) && !gram &&
!want_lp && e.dump_logits.empty();
if (spec) {
struct ServeBase : npu::mtp::BaseModel {
Q4nxNpuForward& f;
Expand All @@ -596,9 +604,20 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
int hidden_size() const override { return int(f.last_hidden().size()); }
int max_batch_rows() const override { return rows; }
bool step(int t, int p, std::vector<float>& l) override { return f.step(t, p, l); }
// The forward's rows are the head's width (M x NV); the loop indexes them at vocab(), the tokenizer's
// bound, which is narrower whenever the head carries padding ids (Qwen3: 151,936 against 151,669). So
// each row is cut to its first nv logits here, in place, before the loop reads row i at i * nv.
bool step_batch_all(const std::vector<int>& t, int s0, std::vector<float>& l) override {
f.begin_verify(int(t.size()), s0);
return f.step_batch_all(t, s0, l);
if (!f.step_batch_all(t, s0, l)) return false;
const size_t m = t.size(), w = m ? l.size() / m : 0;
if (w < size_t(nv) || w * m != l.size()) return false;
if (w != size_t(nv)) {
for (size_t i = 1; i < m; i++)
std::copy_n(l.begin() + std::ptrdiff_t(i * w), nv, l.begin() + std::ptrdiff_t(i * size_t(nv)));
l.resize(m * size_t(nv));
}
return true;
}
bool commit_verify(int a) override { return f.commit_verify_rows(a); }
const std::vector<float>& last_hidden() const override { return f.last_hidden(); }
Expand Down Expand Up @@ -630,20 +649,22 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
e.draft->cfg.block_size, e.draft->cfg.sliding_window);
}
npu::mtp::DraftHead& head = dflash ? static_cast<npu::mtp::DraftHead&>(*dflash) : lookup;
npu::SpecSampler sampler(opts.sample, rng); // used for a sampled request (npu_spec_sampler.h)
npu::mtp::Options o;
o.k = e.draft ? e.spec_draft_k : e.spec_lookup;
o.max_tokens = limit;
o.tau = 0.0f;
o.fallback_below = e.spec_fallback;
o.consume_last = false; // the plain loop does not run the reply's last token either
o.sampler = opts.sample.greedy() ? nullptr : &sampler;
const npu::mtp::Result r = npu::mtp::run_from(
base, &head, int(ids.size()), logits, o, [&](int t) {
if (req.is_connection_closed() || (bridge && bridge->cancelled())) return false;
return deliver(t);
});
if (!r.ok) throw std::runtime_error("speculative decode: " + r.err);
std::fprintf(stderr, "[spec] %s %d tokens: %d rounds, %d of %d drafts accepted, %d plain steps%s\n",
e.draft ? "dflash" : "lookup", int(r.tokens.size()), r.stats.batches, r.stats.accepted, r.stats.drafted, r.stats.steps,
std::fprintf(stderr, "[spec] %s%s %d tokens: %d rounds, %d of %d drafts accepted, %d plain steps%s\n",
e.draft ? "dflash" : "lookup", o.sampler ? " sampled" : "", int(r.tokens.size()), r.stats.batches, r.stats.accepted, r.stats.drafted, r.stats.steps,
r.stats.fell_back_at >= 0 ? (", fell back after round " + std::to_string(r.stats.fell_back_at)).c_str()
: "");
}
Expand Down Expand Up @@ -1155,6 +1176,7 @@ int run_forward_serve(int argc, char** argv) {
int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's)
int spec_lookup = 0; // --spec-lookup K (engine#1781): prompt-lookup drafts, K per verify batch
double spec_fallback = 0.0; // --spec-fallback X: stop drafting below X tokens per round
bool spec_sample = false; // --spec-sample: sampled requests speculate too (distribution-keeping rule)
std::string spec_draft; // --spec-draft <dir>: a DFlash draft package as the drafter
int spec_draft_k = 0; // --spec-draft-k K: drafts per round (default: the package's block - 1)
for (int i = 0; i < argc; i++) {
Expand All @@ -1178,19 +1200,22 @@ int run_forward_serve(int argc, char** argv) {
else if (a == "--dump-logits") dump_logits = next();
else if (a == "--spec-lookup") spec_lookup = std::stoi(next());
else if (a == "--spec-fallback") spec_fallback = std::stod(next());
else if (a == "--spec-sample") spec_sample = true;
else if (a == "--spec-draft") spec_draft = next();
else if (a == "--spec-draft-k") spec_draft_k = std::stoi(next());
else throw std::runtime_error("unknown option " + a);
}
if (model.empty())
throw std::runtime_error("usage: 1bit forward-serve -m <model.q4nx> --kernels <dir> [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits <dir>] [--spec-lookup K | --spec-draft <dir> [--spec-draft-k K]] [--spec-fallback X]");
throw std::runtime_error("usage: 1bit forward-serve -m <model.q4nx> --kernels <dir> [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits <dir>] [--spec-lookup K | --spec-draft <dir> [--spec-draft-k K]] [--spec-fallback X] [--spec-sample]");
if (!spec_draft.empty() && spec_lookup > 0)
throw std::runtime_error("--spec-draft and --spec-lookup are two drafters; give one");
if (spec_draft_k != 0 && spec_draft.empty()) throw std::runtime_error("--spec-draft-k needs --spec-draft");
if (spec_lookup < 0 || spec_lookup > 15)
throw std::runtime_error("--spec-lookup takes 0..15 drafts per round (a verify batch holds at most 16 rows), not " +
std::to_string(spec_lookup));
if (!(spec_fallback >= 0.0)) throw std::runtime_error("--spec-fallback takes a number of tokens >= 0");
if (spec_sample && spec_lookup == 0 && spec_draft.empty())
throw std::runtime_error("--spec-sample needs a drafter (--spec-lookup K or --spec-draft <dir>)");
if (streams < 1 || streams > kNpuMaxStreams)
throw std::runtime_error("--npu-streams takes 1.." + std::to_string(kNpuMaxStreams) +
" (the device's hwctx limit), not " + std::to_string(streams));
Expand All @@ -1212,6 +1237,7 @@ int run_forward_serve(int argc, char** argv) {
e.id = alias.empty() ? std::filesystem::path(dir).filename().string() : alias;
e.spec_lookup = spec_lookup;
e.spec_fallback = spec_fallback;
e.spec_sample = spec_sample;
if (!dump_logits.empty()) {
std::error_code ec;
std::filesystem::create_directories(dump_logits, ec);
Expand Down
78 changes: 56 additions & 22 deletions app/npu_sampling.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,31 +91,13 @@ inline size_t token_bound(size_t width, int vocab) {
return (vocab > 0 && size_t(vocab) < width) ? size_t(vocab) : width;
}

// Draw one token from `logits`. `allowed`, when given, is a per-token mask (non-zero =
// the grammar accepts the token); a mask with nothing set is refused with -1 and the
// caller reports it rather than decoding an unconstrained token. `vocab` is the
// tokenizer's size (token_bound): ids at or past it are never drawn, with or without a mask.
inline int sample_token(const std::vector<float>& logits, const SampleParams& p, std::mt19937_64& rng,
const std::vector<char>* allowed = nullptr, int vocab = 0) {
const size_t n = token_bound(logits.size(), vocab);
std::vector<Candidate> c;
c.reserve(n);
for (size_t i = 0; i < n; i++) {
if (allowed && !(*allowed)[i]) continue;
c.push_back({logits[i], int(i)});
}
if (c.empty()) return -1;
// A higher logit wins; equal logits keep the lower id so the same seed always draws
// the same token.
// What sample_token() draws from when it samples: `c` (the candidates it kept from the logits, a mask applied) is cut by
// top_k and top_p and each survivor's `logit` is replaced by its weight exp((logit - max) / temperature); the total is
// returned. A draw is the first candidate whose running weight reaches u * total. Not for greedy parameters.
inline double sampling_weights(std::vector<Candidate>& c, const SampleParams& p) {
auto better = [](const Candidate& a, const Candidate& b) {
return a.logit > b.logit || (a.logit == b.logit && a.id < b.id);
};

if (p.greedy()) {
// Exactly the argmax the route used before sampling existed.
return std::min_element(c.begin(), c.end(), better)->id;
}

// top_k: keep the k highest logits (llama.cpp llama_sampler_top_k_impl).
if (p.top_k > 0 && size_t(p.top_k) < c.size()) {
std::partial_sort(c.begin(), c.begin() + p.top_k, c.end(), better);
Expand Down Expand Up @@ -158,6 +140,35 @@ inline int sample_token(const std::vector<float>& logits, const SampleParams& p,
x.logit = float(std::exp(double(x.logit - max_l) / double(p.temperature)));
sum += x.logit;
}
return sum;
}

// Draw one token from `logits`. `allowed`, when given, is a per-token mask (non-zero =
// the grammar accepts the token); a mask with nothing set is refused with -1 and the
// caller reports it rather than decoding an unconstrained token. `vocab` is the
// tokenizer's size (token_bound): ids at or past it are never drawn, with or without a mask.
inline int sample_token(const std::vector<float>& logits, const SampleParams& p, std::mt19937_64& rng,
const std::vector<char>* allowed = nullptr, int vocab = 0) {
const size_t n = token_bound(logits.size(), vocab);
std::vector<Candidate> c;
c.reserve(n);
for (size_t i = 0; i < n; i++) {
if (allowed && !(*allowed)[i]) continue;
c.push_back({logits[i], int(i)});
}
if (c.empty()) return -1;
// A higher logit wins; equal logits keep the lower id so the same seed always draws
// the same token.
auto better = [](const Candidate& a, const Candidate& b) {
return a.logit > b.logit || (a.logit == b.logit && a.id < b.id);
};

if (p.greedy()) {
// Exactly the argmax the route used before sampling existed.
return std::min_element(c.begin(), c.end(), better)->id;
}

const double sum = sampling_weights(c, p);
std::uniform_real_distribution<double> dist(0.0, 1.0);
const double target = sum * dist(rng);
double run = 0.0;
Expand All @@ -168,6 +179,29 @@ inline int sample_token(const std::vector<float>& logits, const SampleParams& p,
return c.back().id;
}

// The distribution sample_token() draws from, as one probability per id of the bounded row: zero for every id that
// top_k or top_p cut, w / total for the rest (sampling_weights). Greedy parameters give the one-hot of the argmax. No
// mask: this is what a speculative round checks a draft against (npu/mtp/draft.h RowSampler, engine#1781), and that
// round never runs under a grammar.
inline std::vector<float> sample_probs(const float* logits, size_t width, const SampleParams& p, int vocab = 0) {
const size_t n = token_bound(width, vocab);
std::vector<float> out(n, 0.0f);
if (n == 0) return out;
if (p.greedy()) {
size_t best = 0;
for (size_t i = 1; i < n; i++)
if (logits[i] > logits[best]) best = i;
out[best] = 1.0f;
return out;
}
std::vector<Candidate> c;
c.reserve(n);
for (size_t i = 0; i < n; i++) c.push_back({logits[i], int(i)});
const double sum = sampling_weights(c, p);
for (const auto& x : c) out[size_t(x.id)] = float(double(x.logit) / sum);
return out;
}

// The greedy id of one logits row of `width` floats, bounded like sample_token(): a higher
// logit wins and a tie keeps the lower id. `1bit npu-run` prints this per row.
inline int argmax_token(const float* row, size_t width, int vocab = 0) {
Expand Down
55 changes: 55 additions & 0 deletions app/npu_spec_sampler.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// npu_spec_sampler.h -- a sampled request's choices inside the speculative loop (engine#1781 gate 2): the
// npu/mtp/draft.h RowSampler that `1bit forward-serve --spec-sample` hands mtp::run_from().
//
// pick() is the plain loop's sample_token() on the row. check() keeps a drafted token x with probability p(x), p being
// what sample_token() draws from under the same parameters (sample_probs()), and otherwise emits a draw from p without
// x: npu/mtp/spec_accept.h's rule for a drafter whose own distribution is one-hot at its proposal, which is what the
// prompt lookup and a DFlash head are. Both uniforms of a check come from the request's generator, in order, so a
// reply is reproducible from its seed. The reply follows the plain loop's distribution, not its random stream.
// tests/mtp_sampled_test.cpp holds the distribution, top_k and top_p included.
#pragma once

#include <random>
#include <vector>

#include "../npu/mtp/draft.h"
#include "npu_sampling.h"

namespace onebit::npu {

class SpecSampler : public mtp::RowSampler {
public:
SpecSampler(const SampleParams& p, std::mt19937_64& rng) : sp_(p), rng_(rng) {}
int pick(const float* logits, int n) override {
return sample_token(std::vector<float>(logits, logits + n), sp_, rng_, nullptr, n);
}
mtp::SpecStep check(const float* logits, int n, int x) override {
const std::vector<float> p = sample_probs(logits, size_t(n), sp_, n);
std::vector<float> q(size_t(n), 0.0f);
if (x >= 0 && x < n) q[size_t(x)] = 1.0f;
std::uniform_real_distribution<double> u(0.0, 1.0);
const double ua = u(rng_), ur = u(rng_);
return mtp::spec_accept_step(p.data(), q.data(), n, x, ua, ur);
}

private:
const SampleParams& sp_;
std::mt19937_64& rng_;
};

} // namespace onebit::npu
Loading
Loading