From 3128fa6a56082e6e7f404deb63ebbec3c2fee981 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 02:14:34 -0400 Subject: [PATCH] npu: mtp::run_from() after a batched prefill, the speculation fallback inside the loop, and the probe through both (refs #1781) - mtp::run_from(base, head, pos, logits, opt): the loop from a base that already ran its prompt; run() is a token-by-token prefill and then run_from(). - Options::fallback_below / fallback_window: SpecFallback (spec_greedy.h) recorded after every round, one-pass and two-pass; once off, plain steps. Stats::fell_back_at. - tools/dflash2_probe loop, DFLASH2_MTP_RUN=1: the same prefill, then run_from() with DFlashHead and the DFlash 2 selector as its pick hook (the selector is one lambda, shared with the hand-written loop). tests/mtp_draft_test.cpp [18]: run_from after the caller's prefill gives run()'s tokens, batches and positions; no prefilled position is refused; a never-landing head falls back after exactly 3 rounds on both rounds with the same text; a perfect head never does; rounds of exactly 2 against a bar of 2.0 keep drafting. Four mutants (off ignored, no record on either round, record take instead of take + 1) each fail it. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 8 +++ npu/mtp/draft.h | 42 +++++++++++---- tests/mtp_draft_test.cpp | 55 +++++++++++++++++++ tools/dflash2_probe.cpp | 111 ++++++++++++++++++++++++++++++--------- 4 files changed, 182 insertions(+), 34 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index c0dbf8b2d..99475c43e 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3413,6 +3413,14 @@ selector) are hooks. `tests/dflash_head_test.cpp` runs it through `mtp::run()` o way: the text equals plain decode's, the sink ends holding exactly the committed rows, and every draft equals `draft_forward()` over those rows, bit for bit. +Two pieces for a caller that is not a test. `mtp::run_from()` starts the loop from a base that already ran its prompt +(a batched prefill: the positions it holds and the logits after the last), where `run()` prefills one token at a time. +And `Options::fallback_below` / `fallback_window` apply the speculation fallback (`SpecFallback`, +`npu/forward/include/spec_greedy.h`) inside the loop, on either round: once the last `fallback_window` rounds emitted +fewer than `fallback_below` tokens on average, the run finishes with plain steps (`Stats::fell_back_at` says after which +round). `tools/dflash2_probe loop` with `DFLASH2_MTP_RUN=1` drives the served 27B through exactly these pieces, with +`DFlashHead` and the DFlash 2 selector as its pick hook, after the same batched prefill as its hand-written loop. + **What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth 6 measured 0 %). diff --git a/npu/mtp/draft.h b/npu/mtp/draft.h index 37282990e..df2fd4f89 100644 --- a/npu/mtp/draft.h +++ b/npu/mtp/draft.h @@ -58,6 +58,8 @@ #include #include +#include "../forward/include/spec_greedy.h" + namespace onebit::npu::mtp { // Greedy choice with a deterministic tie-break (first maximum), so a draft that reproduces the model @@ -166,6 +168,11 @@ struct Options { // draft is verified anyway (it is already in the batch), it just does not get followed by more of them. // tau <= 0 disables the gate. float tau = 0.6f; + // The fallback (SpecFallback, npu/forward/include/spec_greedy.h): once the last `fallback_window` rounds emitted + // fewer than `fallback_below` tokens on average, the run stops drafting and finishes with plain steps. + // fallback_below <= 0 turns it off. + double fallback_below = 0.0; + int fallback_window = 3; }; struct Stats { @@ -210,6 +217,8 @@ struct Stats { double ms_draft = 0.0; // the head's chain: its matvecs and the lm head per draft double ms_commit = 0.0; // commit_verify (snapshot restore + replay) and the drafter's row advance double phase_ms() const { return ms_step + ms_batch + ms_draft + ms_commit; } + // The iteration after which the fallback stopped drafting, or -1 when it never did. + int fell_back_at = -1; }; // The live loop's side of the per-depth comparison, built in ONE place so the report sites and the tests cannot @@ -236,14 +245,17 @@ struct Result { std::string err; }; -// Draft, verify, accept the longest prefix, roll back. `on_token` returning false stops the loop. -inline Result run(BaseModel& base, DraftHead* head, const std::vector& prompt, const Options& opt, - const std::function& on_token = {}) { +// The loop from a base that has already run its prompt: `pos` is the number of positions it holds and `logits` the +// choice after the last of them (what a batched prefill returns). run() below is this after a token-by-token prefill. +// `on_token` returning false stops the loop. +inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector logits, const Options& opt, + const std::function& on_token = {}) { Result r; const int V = base.vocab(); if (V <= 0) { r.err = "no vocabulary"; return r; } - if (prompt.empty()) { r.err = "empty prompt"; return r; } + if (pos <= 0 || (int)logits.size() < V) { r.err = "no prefilled prompt"; return r; } const int k = head ? std::min(std::max(0, opt.k), base.max_batch_rows()) : 0; + onebit::npu::SpecFallback fallback(opt.fallback_below, opt.fallback_window); // Phase timing: always accumulated (steady_clock::now() is ~20 ns against a ~0.5 s pass), printed by the // caller when it wants the split. Measurement only -- nothing here is read back into the loop's decisions. @@ -256,11 +268,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector& prom }; Clock::time_point tp = Clock::now(); - std::vector logits, all; - int pos = 0; - for (int t : prompt) - if (!base.step(t, pos++, logits)) { r.err = "prefill: step failed"; return r; } - + std::vector all; auto emit = [&](int t) { r.tokens.push_back(t); return !on_token || on_token(t); }; while ((int)r.tokens.size() < opt.max_tokens) { @@ -270,7 +278,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector& prom const bool keep_going = emit(t1); if (!keep_going) break; - if (k <= 0) { // plain decode + if (k <= 0 || fallback.off()) { // plain decode (no drafter, k = 0, or the fallback stopped it) if (!base.step(t1, pos++, logits)) { r.err = "step failed"; return r; } r.stats.steps++; r.stats.ms_step += lap(tp); @@ -351,6 +359,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector& prom r.stats.ms_commit += lap(tp); logits.assign(all.begin() + (size_t)take * (size_t)V, all.begin() + (size_t)(take + 1) * (size_t)V); pos += take + 1; + if (fallback.record(take + 1) && r.stats.fell_back_at < 0) r.stats.fell_back_at = r.stats.iterations; if (stopped || (int)r.tokens.size() >= opt.max_tokens) break; continue; } @@ -451,6 +460,7 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector& prom logits.assign(all.begin() + (size_t)(take - 1) * (size_t)V, all.begin() + (size_t)take * (size_t)V); pos = have + take; + if (fallback.record(take + 1) && r.stats.fell_back_at < 0) r.stats.fell_back_at = r.stats.iterations; if (stopped || (int)r.tokens.size() >= opt.max_tokens) break; } @@ -459,6 +469,18 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector& prom return r; } +// Draft, verify, accept the longest prefix, roll back, after running `prompt` one token at a time. +inline Result run(BaseModel& base, DraftHead* head, const std::vector& prompt, const Options& opt, + const std::function& on_token = {}) { + if (base.vocab() <= 0) { Result r; r.err = "no vocabulary"; return r; } + if (prompt.empty()) { Result r; r.err = "empty prompt"; return r; } + std::vector logits; + int pos = 0; + for (int t : prompt) + if (!base.step(t, pos++, logits)) { Result r; r.err = "prefill: step failed"; return r; } + return run_from(base, head, pos, std::move(logits), opt, on_token); +} + // Plain greedy decode through the same interface, so the test (and a --mtp-k 0 run) can compare text. inline Result run_plain(BaseModel& base, const std::vector& prompt, const Options& opt, const std::function& on_token = {}) { diff --git a/tests/mtp_draft_test.cpp b/tests/mtp_draft_test.cpp index 9ccb5e7b9..dc6dd6a6f 100644 --- a/tests/mtp_draft_test.cpp +++ b/tests/mtp_draft_test.cpp @@ -1034,6 +1034,61 @@ int main() { "one-pass with max_batch_rows() == 1: plain steps, no batch, the drafter still sees every row"); } + // [18] run_from(): the loop after a caller's own prefill gives run()'s result; and the fallback stops drafting + // after the window it is given, on both rounds, without changing the text. + { + Options o = base_opt; + o.k = 4; + ToyBase a(8), b(8); + OnePassHead ha(OnePassHead::kHalf, (int)P.size()), hb(OnePassHead::kHalf, (int)P.size()); + const Result ra = onebit::npu::mtp::run(a, &ha, P, o); + std::vector lg; + int pos = 0; + for (int t : P) b.step(t, pos++, lg); + const Result rb = onebit::npu::mtp::run_from(b, &hb, pos, lg, o); + check(ra.ok && rb.ok && same(ra.tokens, rb.tokens) && ra.stats.batches == rb.stats.batches && + a.consumed() == b.consumed(), + "run_from after the caller's prefill: run()'s tokens, batches and positions"); + ToyBase c(8); + OnePassHead hc(OnePassHead::kHalf, 0); + check(!onebit::npu::mtp::run_from(c, &hc, 0, std::vector(V, 0.0f), o).ok, + "run_from with no prefilled position is refused"); + + for (int one_pass : {1, 0}) { + ToyBase m(8); + OnePassHead og(OnePassHead::kGarbage, (int)P.size()); + GarbageHead tg; + Options f = base_opt; + f.k = 4; + f.tau = 0.0f; + f.fallback_below = 2.0; + f.fallback_window = 3; + DraftHead* h = one_pass ? (DraftHead*)&og : (DraftHead*)&tg; + const Result r = onebit::npu::mtp::run(m, h, P, f); + char msg[200]; + std::snprintf(msg, sizeof msg, + "fallback (%s round): a head that never lands stops after 3 rounds (fell back at %d, %d batches), " + "same text", + one_pass ? "one-pass" : "two-pass", r.stats.fell_back_at, m.batches()); + check(r.ok && same(r.tokens, ref.tokens) && r.stats.fell_back_at == 3 && m.batches() == 3 && + m.consumed() == (int)P.size() + WANT, + msg); + } + ToyBase m(8); + OnePassHead op(OnePassHead::kPerfect, (int)P.size()); + Options f = base_opt; + f.k = 4; + f.fallback_below = 2.0; + const Result r = onebit::npu::mtp::run(m, &op, P, f); + check(r.ok && same(r.tokens, ref.tokens) && r.stats.fell_back_at == -1, + "fallback: a head that lands every draft never falls back"); + ToyBase eq(8); + OnePassHead oh(OnePassHead::kHalf, (int)P.size()); // every round emits exactly 2: at the bar, not under it + const Result re = onebit::npu::mtp::run(eq, &oh, P, f); + check(re.ok && same(re.tokens, ref.tokens) && re.stats.fell_back_at == -1, + "fallback: rounds of exactly 2 tokens against a bar of 2.0 keep drafting"); + } + std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails); return fails ? 1 : 0; } diff --git a/tools/dflash2_probe.cpp b/tools/dflash2_probe.cpp index 571258389..6bb24526a 100644 --- a/tools/dflash2_probe.cpp +++ b/tools/dflash2_probe.cpp @@ -50,6 +50,7 @@ #include "dflash_draft.h" #include "dflash_forward.h" +#include "dflash_head.h" // DFLASH2_MTP_RUN: the same loop through mtp::run_from + DFlashHead #include "file_create.h" // onebit_fopen_private_wb #include "file_map.h" #include "forward/include/q4nx_dequant.h" @@ -397,6 +398,63 @@ void dev_place(DevDraft& d, xrt::device& dev, const std::string& kdir, const std d.proj[W.data()] = std::move(p); } +// DFLASH2_MTP_RUN=1: the loop after the same prefill, through the engine's own pieces -- mtp::run_from() with +// DFlashHead as the drafter (npu/mtp/draft.h, npu/dflash_head.h) -- instead of the hand-written rounds below, so a +// ticket can hold the production path against the reference loop and plain decode. DFLASH2_FALLBACK and +// DFLASH2_FALLBACK_ROUNDS are the same rule, through Options::fallback_*. +struct FwdBase : onebit::npu::mtp::BaseModel { + Q4nxNpuForward& f; + int nv, h; + FwdBase(Q4nxNpuForward& fw, int vocab, int hidden) : f(fw), nv(vocab), h(hidden) {} + int vocab() const override { return nv; } + int hidden_size() const override { return h; } + int max_batch_rows() const override { return f.max_batch_rows(); } + bool step(int t, int p, std::vector& l) override { return f.step(t, p, l); } + bool step_batch_all(const std::vector& t, int s, std::vector& l) override { + f.begin_verify(int(t.size()), s); + return f.step_batch_all(t, s, l); + } + bool commit_verify(int a) override { return f.commit_verify_rows(a); } + const std::vector& last_hidden() const override { return f.last_hidden(); } + const std::vector& last_hidden_rows() const override { return f.last_hidden_rows(); } +}; + +template +int loop_mtp_run(Q4nxNpuForward& fwd, const DraftWeights& w, const DraftConfig& dc, std::vector& taps, + const std::vector& prompt, const std::vector& logits, const std::vector& ref, + int want_tokens, int per, double t_prefill, Select& select) { + const int P = int(prompt.size()); + HeadHooks hooks; + hooks.embed = [&](int t, std::vector& row) { fwd.embed_row(t, row); return !row.empty(); }; + hooks.pick = [&](const float* h, int prev) { return select(h, prev); }; + DFlashHead head(w, taps, hooks, int(dc.mask_token_id), dc.block_size, dc.sliding_window); + FwdBase base(fwd, int(logits.size()), dc.hidden); + onebit::npu::mtp::Options o; + o.k = per; + o.max_tokens = want_tokens; + o.tau = 0.0f; + o.fallback_below = getenv("DFLASH2_FALLBACK") ? std::atof(getenv("DFLASH2_FALLBACK")) : 0.0; + o.fallback_window = getenv("DFLASH2_FALLBACK_ROUNDS") ? std::max(1, std::atoi(getenv("DFLASH2_FALLBACK_ROUNDS"))) : 3; + const double t0 = now_s(); + const onebit::npu::mtp::Result r = onebit::npu::mtp::run_from(base, &head, P, logits, o); + const double t_gen = now_s() - t0; + if (!r.ok) die("mtp_run: " + r.err + (head.error().empty() ? "" : " (" + head.error() + ")")); + const int n = int(r.tokens.size()); + int same = 0; + if (!ref.empty()) + while (same < n && P + same < int(ref.size()) && ref[size_t(P + same)] == r.tokens[size_t(same)]) same++; + std::printf("mtp_run: prefill %d tokens in %.2f s; %d tokens in %d rounds + %d steps, %.3f accepted per round, %.3f base passes per token%s\n", + P, t_prefill, n, r.stats.batches, r.stats.steps, r.stats.mean_accept_len(), r.stats.passes_per_token(), + r.stats.fell_back_at >= 0 ? (", fell back after round " + std::to_string(r.stats.fell_back_at)).c_str() : ""); + std::printf("mtp_run: generation %.2f s = %.2f tok/s; draft %.0f ms, verify %.0f ms, commit %.0f ms, steps %.0f ms in all\n", + t_gen, double(n) / t_gen, r.stats.ms_draft, r.stats.ms_batch, r.stats.ms_commit, r.stats.ms_step); + if (!ref.empty()) std::printf("mtp_run: the first %d of %d tokens equal plain greedy decoding%s\n", same, n, same == n ? "" : " (then they part)"); + std::printf("tokens:"); + for (int i = 0; i < n; i++) std::printf(" %d", r.tokens[size_t(i)]); + std::printf("\n"); + return 0; +} + int loop(int argc, char** argv) { if (argc != 8) die("loop "); const std::string model = argv[2], kern = argv[3], ddir = argv[4]; @@ -459,6 +517,34 @@ int loop(int argc, char** argv) { std::vector mask_emb; fwd.embed_row(int(dc.mask_token_id), mask_emb); double t_draft = 0, t_head = 0, t_verify = 0, t_fuse = 0; + // The selector: the target's head on one drafted row, then the pairwise re-score against the previous pick. + std::vector prow(R), hp(R), succ(size_t(k) * R); + auto select = [&](const float* h, int prev) -> int { + std::vector hv(h, h + H), lg; + double b0 = now_s(); + if (!fwd.lm_head_logits(hv, lg)) die(std::string("head: ") + fwd.last_error()); + t_head += now_s() - b0; + b0 = now_s(); + std::vector idx(lg.size()); + for (size_t j = 0; j < idx.size(); j++) idx[j] = int(j); + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return lg[size_t(a)] > lg[size_t(b)]; }); + for (size_t r = 0; r < R; r++) { + double acc = 0.0; + const float* wr = hproj.data() + r * H; + for (size_t q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]); + hp[r] = float(acc); + } + decode_row(dm.selector_pred, prev, prow.data()); + std::vector unary(static_cast(k)); + for (int c = 0; c < k; c++) { + unary[size_t(c)] = lg[size_t(idx[size_t(c)])]; + decode_row(dm.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * R); + } + const int pick = idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), int(R)))]; + t_draft += now_s() - b0; + return pick; + }; + if (getenv("DFLASH2_MTP_RUN")) return loop_mtp_run(fwd, w, dc, taps, prompt, logits, ref, want_tokens, per, t_prefill, select); long rounds = 0, accepted_total = 0; std::vector hist(size_t(block), 0); const double t_gen0 = now_s(); @@ -496,33 +582,10 @@ int loop(int argc, char** argv) { // selector: the target's head on each drafted row, then the pairwise re-score against the previous pick std::vector drafts; int prev = cur; - std::vector prow(R), hp(R), succ(size_t(k) * R); for (int i = 1; i <= per; i++) { - const float* h = hrows.data() + size_t(i) * H; - std::vector hv(h, h + H), lg; - double b0 = now_s(); - if (!fwd.lm_head_logits(hv, lg)) die(std::string("head: ") + fwd.last_error()); - t_head += now_s() - b0; - b0 = now_s(); - std::vector idx(lg.size()); - for (size_t j = 0; j < idx.size(); j++) idx[j] = int(j); - std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { return lg[size_t(a)] > lg[size_t(b)]; }); - for (size_t r = 0; r < R; r++) { - double acc = 0.0; - const float* wr = hproj.data() + r * H; - for (size_t q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]); - hp[r] = float(acc); - } - decode_row(dm.selector_pred, prev, prow.data()); - std::vector unary(static_cast(k)); - for (int c = 0; c < k; c++) { - unary[size_t(c)] = lg[size_t(idx[size_t(c)])]; - decode_row(dm.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * R); - } - const int pick = idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), int(R)))]; + const int pick = select(hrows.data() + size_t(i) * H, prev); drafts.push_back(pick); prev = pick; - t_draft += now_s() - b0; } // verify: [cur, d1..d7] in one batched pass; row i predicts the token after position pos + i double c0 = now_s();