Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -3413,6 +3413,14 @@ selector) are hooks. `tests/dflash_head_test.cpp` runs it through `mtp::run()` o
way: the text equals plain decode's, the sink ends holding exactly the committed rows, and every draft equals
`draft_forward()` over those rows, bit for bit.

Two pieces for a caller that is not a test. `mtp::run_from()` starts the loop from a base that already ran its prompt
(a batched prefill: the positions it holds and the logits after the last), where `run()` prefills one token at a time.
And `Options::fallback_below` / `fallback_window` apply the speculation fallback (`SpecFallback`,
`npu/forward/include/spec_greedy.h`) inside the loop, on either round: once the last `fallback_window` rounds emitted
fewer than `fallback_below` tokens on average, the run finishes with plain steps (`Stats::fell_back_at` says after which
round). `tools/dflash2_probe loop` with `DFLASH2_MTP_RUN=1` drives the served 27B through exactly these pieces, with
`DFlashHead` and the DFlash 2 selector as its pick hook, after the same batched prefill as its hand-written loop.

**What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel
lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth
6 measured 0 %).
Expand Down
42 changes: 32 additions & 10 deletions npu/mtp/draft.h
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@
#include <string>
#include <vector>

#include "../forward/include/spec_greedy.h"

namespace onebit::npu::mtp {

// Greedy choice with a deterministic tie-break (first maximum), so a draft that reproduces the model
Expand Down Expand Up @@ -166,6 +168,11 @@ struct Options {
// draft is verified anyway (it is already in the batch), it just does not get followed by more of them.
// tau <= 0 disables the gate.
float tau = 0.6f;
// The fallback (SpecFallback, npu/forward/include/spec_greedy.h): once the last `fallback_window` rounds emitted
// fewer than `fallback_below` tokens on average, the run stops drafting and finishes with plain steps.
// fallback_below <= 0 turns it off.
double fallback_below = 0.0;
int fallback_window = 3;
};

struct Stats {
Expand Down Expand Up @@ -210,6 +217,8 @@ struct Stats {
double ms_draft = 0.0; // the head's chain: its matvecs and the lm head per draft
double ms_commit = 0.0; // commit_verify (snapshot restore + replay) and the drafter's row advance
double phase_ms() const { return ms_step + ms_batch + ms_draft + ms_commit; }
// The iteration after which the fallback stopped drafting, or -1 when it never did.
int fell_back_at = -1;
};

// The live loop's side of the per-depth comparison, built in ONE place so the report sites and the tests cannot
Expand All @@ -236,14 +245,17 @@ struct Result {
std::string err;
};

// Draft, verify, accept the longest prefix, roll back. `on_token` returning false stops the loop.
inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prompt, const Options& opt,
const std::function<bool(int)>& on_token = {}) {
// The loop from a base that has already run its prompt: `pos` is the number of positions it holds and `logits` the
// choice after the last of them (what a batched prefill returns). run() below is this after a token-by-token prefill.
// `on_token` returning false stops the loop.
inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector<float> logits, const Options& opt,
const std::function<bool(int)>& on_token = {}) {
Result r;
const int V = base.vocab();
if (V <= 0) { r.err = "no vocabulary"; return r; }
if (prompt.empty()) { r.err = "empty prompt"; return r; }
if (pos <= 0 || (int)logits.size() < V) { r.err = "no prefilled prompt"; return r; }
const int k = head ? std::min(std::max(0, opt.k), base.max_batch_rows()) : 0;
onebit::npu::SpecFallback fallback(opt.fallback_below, opt.fallback_window);

// Phase timing: always accumulated (steady_clock::now() is ~20 ns against a ~0.5 s pass), printed by the
// caller when it wants the split. Measurement only -- nothing here is read back into the loop's decisions.
Expand All @@ -256,11 +268,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prom
};
Clock::time_point tp = Clock::now();

std::vector<float> logits, all;
int pos = 0;
for (int t : prompt)
if (!base.step(t, pos++, logits)) { r.err = "prefill: step failed"; return r; }

std::vector<float> all;
auto emit = [&](int t) { r.tokens.push_back(t); return !on_token || on_token(t); };

while ((int)r.tokens.size() < opt.max_tokens) {
Expand All @@ -270,7 +278,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prom
const bool keep_going = emit(t1);
if (!keep_going) break;

if (k <= 0) { // plain decode
if (k <= 0 || fallback.off()) { // plain decode (no drafter, k = 0, or the fallback stopped it)
if (!base.step(t1, pos++, logits)) { r.err = "step failed"; return r; }
r.stats.steps++;
r.stats.ms_step += lap(tp);
Expand Down Expand Up @@ -351,6 +359,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prom
r.stats.ms_commit += lap(tp);
logits.assign(all.begin() + (size_t)take * (size_t)V, all.begin() + (size_t)(take + 1) * (size_t)V);
pos += take + 1;
if (fallback.record(take + 1) && r.stats.fell_back_at < 0) r.stats.fell_back_at = r.stats.iterations;
if (stopped || (int)r.tokens.size() >= opt.max_tokens) break;
continue;
}
Expand Down Expand Up @@ -451,6 +460,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prom
logits.assign(all.begin() + (size_t)(take - 1) * (size_t)V,
all.begin() + (size_t)take * (size_t)V);
pos = have + take;
if (fallback.record(take + 1) && r.stats.fell_back_at < 0) r.stats.fell_back_at = r.stats.iterations;
if (stopped || (int)r.tokens.size() >= opt.max_tokens) break;
}

Expand All @@ -459,6 +469,18 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prom
return r;
}

// Draft, verify, accept the longest prefix, roll back, after running `prompt` one token at a time.
inline Result run(BaseModel& base, DraftHead* head, const std::vector<int>& prompt, const Options& opt,
const std::function<bool(int)>& on_token = {}) {
if (base.vocab() <= 0) { Result r; r.err = "no vocabulary"; return r; }
if (prompt.empty()) { Result r; r.err = "empty prompt"; return r; }
std::vector<float> logits;
int pos = 0;
for (int t : prompt)
if (!base.step(t, pos++, logits)) { Result r; r.err = "prefill: step failed"; return r; }
return run_from(base, head, pos, std::move(logits), opt, on_token);
}

// Plain greedy decode through the same interface, so the test (and a --mtp-k 0 run) can compare text.
inline Result run_plain(BaseModel& base, const std::vector<int>& prompt, const Options& opt,
const std::function<bool(int)>& on_token = {}) {
Expand Down
55 changes: 55 additions & 0 deletions tests/mtp_draft_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1034,6 +1034,61 @@ int main() {
"one-pass with max_batch_rows() == 1: plain steps, no batch, the drafter still sees every row");
}

// [18] run_from(): the loop after a caller's own prefill gives run()'s result; and the fallback stops drafting
// after the window it is given, on both rounds, without changing the text.
{
Options o = base_opt;
o.k = 4;
ToyBase a(8), b(8);
OnePassHead ha(OnePassHead::kHalf, (int)P.size()), hb(OnePassHead::kHalf, (int)P.size());
const Result ra = onebit::npu::mtp::run(a, &ha, P, o);
std::vector<float> lg;
int pos = 0;
for (int t : P) b.step(t, pos++, lg);
const Result rb = onebit::npu::mtp::run_from(b, &hb, pos, lg, o);
check(ra.ok && rb.ok && same(ra.tokens, rb.tokens) && ra.stats.batches == rb.stats.batches &&
a.consumed() == b.consumed(),
"run_from after the caller's prefill: run()'s tokens, batches and positions");
ToyBase c(8);
OnePassHead hc(OnePassHead::kHalf, 0);
check(!onebit::npu::mtp::run_from(c, &hc, 0, std::vector<float>(V, 0.0f), o).ok,
"run_from with no prefilled position is refused");

for (int one_pass : {1, 0}) {
ToyBase m(8);
OnePassHead og(OnePassHead::kGarbage, (int)P.size());
GarbageHead tg;
Options f = base_opt;
f.k = 4;
f.tau = 0.0f;
f.fallback_below = 2.0;
f.fallback_window = 3;
DraftHead* h = one_pass ? (DraftHead*)&og : (DraftHead*)&tg;
const Result r = onebit::npu::mtp::run(m, h, P, f);
char msg[200];
std::snprintf(msg, sizeof msg,
"fallback (%s round): a head that never lands stops after 3 rounds (fell back at %d, %d batches), "
"same text",
one_pass ? "one-pass" : "two-pass", r.stats.fell_back_at, m.batches());
check(r.ok && same(r.tokens, ref.tokens) && r.stats.fell_back_at == 3 && m.batches() == 3 &&
m.consumed() == (int)P.size() + WANT,
msg);
}
ToyBase m(8);
OnePassHead op(OnePassHead::kPerfect, (int)P.size());
Options f = base_opt;
f.k = 4;
f.fallback_below = 2.0;
const Result r = onebit::npu::mtp::run(m, &op, P, f);
check(r.ok && same(r.tokens, ref.tokens) && r.stats.fell_back_at == -1,
"fallback: a head that lands every draft never falls back");
ToyBase eq(8);
OnePassHead oh(OnePassHead::kHalf, (int)P.size()); // every round emits exactly 2: at the bar, not under it
const Result re = onebit::npu::mtp::run(eq, &oh, P, f);
check(re.ok && same(re.tokens, ref.tokens) && re.stats.fell_back_at == -1,
"fallback: rounds of exactly 2 tokens against a bar of 2.0 keep drafting");
}

std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails);
return fails ? 1 : 0;
}
111 changes: 87 additions & 24 deletions tools/dflash2_probe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@

#include "dflash_draft.h"
#include "dflash_forward.h"
#include "dflash_head.h" // DFLASH2_MTP_RUN: the same loop through mtp::run_from + DFlashHead
#include "file_create.h" // onebit_fopen_private_wb
#include "file_map.h"
#include "forward/include/q4nx_dequant.h"
Expand Down Expand Up @@ -397,6 +398,63 @@ void dev_place(DevDraft& d, xrt::device& dev, const std::string& kdir, const std
d.proj[W.data()] = std::move(p);
}

// DFLASH2_MTP_RUN=1: the loop after the same prefill, through the engine's own pieces -- mtp::run_from() with
// DFlashHead as the drafter (npu/mtp/draft.h, npu/dflash_head.h) -- instead of the hand-written rounds below, so a
// ticket can hold the production path against the reference loop and plain decode. DFLASH2_FALLBACK and
// DFLASH2_FALLBACK_ROUNDS are the same rule, through Options::fallback_*.
struct FwdBase : onebit::npu::mtp::BaseModel {
Q4nxNpuForward& f;
int nv, h;
FwdBase(Q4nxNpuForward& fw, int vocab, int hidden) : f(fw), nv(vocab), h(hidden) {}
int vocab() const override { return nv; }
int hidden_size() const override { return h; }
int max_batch_rows() const override { return f.max_batch_rows(); }
bool step(int t, int p, std::vector<float>& l) override { return f.step(t, p, l); }
bool step_batch_all(const std::vector<int>& t, int s, std::vector<float>& l) override {
f.begin_verify(int(t.size()), s);
return f.step_batch_all(t, s, l);
}
bool commit_verify(int a) override { return f.commit_verify_rows(a); }
const std::vector<float>& last_hidden() const override { return f.last_hidden(); }
const std::vector<float>& last_hidden_rows() const override { return f.last_hidden_rows(); }
};

template <class Select>
int loop_mtp_run(Q4nxNpuForward& fwd, const DraftWeights& w, const DraftConfig& dc, std::vector<float>& taps,
const std::vector<int>& prompt, const std::vector<float>& logits, const std::vector<int>& ref,
int want_tokens, int per, double t_prefill, Select& select) {
const int P = int(prompt.size());
HeadHooks hooks;
hooks.embed = [&](int t, std::vector<float>& row) { fwd.embed_row(t, row); return !row.empty(); };
hooks.pick = [&](const float* h, int prev) { return select(h, prev); };
DFlashHead head(w, taps, hooks, int(dc.mask_token_id), dc.block_size, dc.sliding_window);
FwdBase base(fwd, int(logits.size()), dc.hidden);
onebit::npu::mtp::Options o;
o.k = per;
o.max_tokens = want_tokens;
o.tau = 0.0f;
o.fallback_below = getenv("DFLASH2_FALLBACK") ? std::atof(getenv("DFLASH2_FALLBACK")) : 0.0;
o.fallback_window = getenv("DFLASH2_FALLBACK_ROUNDS") ? std::max(1, std::atoi(getenv("DFLASH2_FALLBACK_ROUNDS"))) : 3;
const double t0 = now_s();
const onebit::npu::mtp::Result r = onebit::npu::mtp::run_from(base, &head, P, logits, o);
const double t_gen = now_s() - t0;
if (!r.ok) die("mtp_run: " + r.err + (head.error().empty() ? "" : " (" + head.error() + ")"));
const int n = int(r.tokens.size());
int same = 0;
if (!ref.empty())
while (same < n && P + same < int(ref.size()) && ref[size_t(P + same)] == r.tokens[size_t(same)]) same++;
std::printf("mtp_run: prefill %d tokens in %.2f s; %d tokens in %d rounds + %d steps, %.3f accepted per round, %.3f base passes per token%s\n",
P, t_prefill, n, r.stats.batches, r.stats.steps, r.stats.mean_accept_len(), r.stats.passes_per_token(),
r.stats.fell_back_at >= 0 ? (", fell back after round " + std::to_string(r.stats.fell_back_at)).c_str() : "");
std::printf("mtp_run: generation %.2f s = %.2f tok/s; draft %.0f ms, verify %.0f ms, commit %.0f ms, steps %.0f ms in all\n",
t_gen, double(n) / t_gen, r.stats.ms_draft, r.stats.ms_batch, r.stats.ms_commit, r.stats.ms_step);
if (!ref.empty()) std::printf("mtp_run: the first %d of %d tokens equal plain greedy decoding%s\n", same, n, same == n ? "" : " (then they part)");
std::printf("tokens:");
for (int i = 0; i < n; i++) std::printf(" %d", r.tokens[size_t(i)]);
std::printf("\n");
return 0;
}

int loop(int argc, char** argv) {
if (argc != 8) die("loop <model dir> <kernel dir> <draft dir> <prompt-ids file> <tokens> <reference seq.i32 or ->");
const std::string model = argv[2], kern = argv[3], ddir = argv[4];
Expand Down Expand Up @@ -459,6 +517,34 @@ int loop(int argc, char** argv) {
std::vector<float> mask_emb;
fwd.embed_row(int(dc.mask_token_id), mask_emb);
double t_draft = 0, t_head = 0, t_verify = 0, t_fuse = 0;
// The selector: the target's head on one drafted row, then the pairwise re-score against the previous pick.
std::vector<float> prow(R), hp(R), succ(size_t(k) * R);
auto select = [&](const float* h, int prev) -> int {
std::vector<float> hv(h, h + H), lg;
double b0 = now_s();
if (!fwd.lm_head_logits(hv, lg)) die(std::string("head: ") + fwd.last_error());
t_head += now_s() - b0;
b0 = now_s();
std::vector<int> idx(lg.size());
for (size_t j = 0; j < idx.size(); j++) idx[j] = int(j);
std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return lg[size_t(a)] > lg[size_t(b)]; });
for (size_t r = 0; r < R; r++) {
double acc = 0.0;
const float* wr = hproj.data() + r * H;
for (size_t q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]);
hp[r] = float(acc);
}
decode_row(dm.selector_pred, prev, prow.data());
std::vector<float> unary(static_cast<size_t>(k));
for (int c = 0; c < k; c++) {
unary[size_t(c)] = lg[size_t(idx[size_t(c)])];
decode_row(dm.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * R);
}
const int pick = idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), int(R)))];
t_draft += now_s() - b0;
return pick;
};
if (getenv("DFLASH2_MTP_RUN")) return loop_mtp_run(fwd, w, dc, taps, prompt, logits, ref, want_tokens, per, t_prefill, select);
long rounds = 0, accepted_total = 0;
std::vector<long> hist(size_t(block), 0);
const double t_gen0 = now_s();
Expand Down Expand Up @@ -496,33 +582,10 @@ int loop(int argc, char** argv) {
// selector: the target's head on each drafted row, then the pairwise re-score against the previous pick
std::vector<int> drafts;
int prev = cur;
std::vector<float> prow(R), hp(R), succ(size_t(k) * R);
for (int i = 1; i <= per; i++) {
const float* h = hrows.data() + size_t(i) * H;
std::vector<float> hv(h, h + H), lg;
double b0 = now_s();
if (!fwd.lm_head_logits(hv, lg)) die(std::string("head: ") + fwd.last_error());
t_head += now_s() - b0;
b0 = now_s();
std::vector<int> idx(lg.size());
for (size_t j = 0; j < idx.size(); j++) idx[j] = int(j);
std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return lg[size_t(a)] > lg[size_t(b)]; });
for (size_t r = 0; r < R; r++) {
double acc = 0.0;
const float* wr = hproj.data() + r * H;
for (size_t q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]);
hp[r] = float(acc);
}
decode_row(dm.selector_pred, prev, prow.data());
std::vector<float> unary(static_cast<size_t>(k));
for (int c = 0; c < k; c++) {
unary[size_t(c)] = lg[size_t(idx[size_t(c)])];
decode_row(dm.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * R);
}
const int pick = idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), int(R)))];
const int pick = select(hrows.data() + size_t(i) * H, prev);
drafts.push_back(pick);
prev = pick;
t_draft += now_s() - b0;
}
// verify: [cur, d1..d7] in one batched pass; row i predicts the token after position pos + i
double c0 = now_s();
Expand Down
Loading