From b14db856779998c7ecdb1b21d876da74ecbd8e89 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 01:08:30 -0400 Subject: [PATCH] npu: mtp::run's one-pass round for a head that does not read the hidden after t1 (refs #1781) A DFlash draft drafts from the base's committed rows plus the last token's embedding, so it does not need the base's hidden after t1. Such a head returns false from DraftHead::needs_last_hidden(), and run() then verifies [t1, d1..dk] as one batch of k + 1 rows instead of a step plus a k-row batch: draft i is checked against row i, the round commits t1 plus the accepted drafts, and the drafter's commit_rows() gets those rows. k is held to max_batch_rows() - 1. The default (true) keeps the two-pass round the MTP head needs. tests/mtp_draft_test.cpp [17]: the text, recurrent state and positions equal plain decode's for perfect, garbage and half-right heads at k = 1..12, both documented row-count shapes; a fully accepted chain takes ceil(N / (k + 1)) batches and no step (k = 7: 0.125 base passes per token against 0.250); every committed row reaches the drafter once, in order; a consumer stop and a one-row batch. Seven mutants each fail it. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 12 ++++ npu/mtp/draft.h | 84 ++++++++++++++++++++++++++ tests/mtp_draft_test.cpp | 124 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 63c28eb7c..95187da73 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3362,6 +3362,18 @@ to 6.9 %** — the head needs the hidden of the token it drafts past, which a on the hidden without lagging the cursor is worse: it accepts **0 of 92** while the mismatch counter stays at 0, i.e. a silently broken loop rather than a slow one. +**The one-pass round, for a head that does not need that hidden** (engine#1781). The refutation above is about the +MTP head. A DFlash draft drafts from the base's *committed* rows plus the last token's embedding, so it never needs the +hidden after the token it drafts past. Such a head says so (`DraftHead::needs_last_hidden()` returns false), and +`mtp::run()` then verifies `[t1, d1..dk]` as one batch of `k + 1` rows: row `i` checks draft `i + 1`, the round commits +`t1` plus the accepted drafts, the drafter's `commit_rows()` gets those rows (`t1`'s first), and the next `t1` is the +base's choice in the first row past them. `k` is held to `max_batch_rows() - 1`, because `t1` takes a row. The MTP head +keeps the two-pass round (the default). `tests/mtp_draft_test.cpp` [17] pins it on the toy base: the text and the +recurrent state are plain decode's for perfect, garbage and half-right heads at `k` = 1 to 12; a fully accepted chain +takes one pass per `k + 1` tokens and no step (k = 7: 0.125 base passes per token against the two-pass round's 0.250); +and every committed row reaches the drafter once, in order. This is what the "one pass (verify only)" rows of +[docs/oga-drafter-path.md](oga-drafter-path.md) price. + **What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth 6 measured 0 %). diff --git a/npu/mtp/draft.h b/npu/mtp/draft.h index 40d93a5de..37282990e 100644 --- a/npu/mtp/draft.h +++ b/npu/mtp/draft.h @@ -144,6 +144,12 @@ class DraftHead { virtual int draft(const std::vector& hidden, int last_token, int pos, int k, std::vector& out, std::vector* conf = nullptr) = 0; virtual void reset() {} + // Whether draft() reads `hidden`, the base's state after `last_token`. true (the MTP head): the loop has to run + // `last_token` through the base first, so a round is two base passes (that step, then the verify batch). + // false (a DFlash draft, which drafts from the base's committed rows plus the last token's embedding): the loop + // verifies [last_token, d1..dk] as ONE batch of k + 1 rows, so a round is one base pass, and draft() is handed an + // empty `hidden`. engine#1781; docs/npu.md, "The draft loop, and whether to serve it". + virtual bool needs_last_hidden() const { return true; } // Called once per iteration with the rows that were committed: `hidden_rows` is M x hidden_size and holds the // base's post-norm hidden of each accepted position (row 0 = the first accepted draft). A drafter keeps its // own stream (the MTP layer attends over its own states), so it advances here -- without this it would see @@ -271,6 +277,84 @@ inline Result run(BaseModel& base, DraftHead* head, const std::vector& prom continue; } + if (!head->needs_last_hidden()) { + // The one-pass round: t1 is row 0 of the verify batch instead of a step of its own. Row i of the batch is + // the base's choice after position pos + i, so draft i is checked against row i, and the round commits t1 + // plus the accepted drafts (take + 1 rows); the next round's t1 is the choice in row `take`. + const int left = opt.max_tokens - (int)r.tokens.size(); // tokens this run may still emit + const int room = std::min({k, base.max_batch_rows() - 1, left}); + std::vector d; + std::vector conf; + int nd = 0; + if (room > 0) { + static const std::vector no_hidden; + r.stats.chain_at.push_back(pos + 1); + nd = head->draft(no_hidden, t1, pos + 1, room, d, &conf); + r.stats.ms_draft += lap(tp); + nd = std::min(nd, std::min(room, (int)d.size())); + } + if (nd <= 0) { + // Nothing to verify (no proposal, or the budget is spent): consume t1 the plain way, so the base has + // run every token it was handed. + if (!base.step(t1, pos++, logits)) { r.err = "step failed"; return r; } + r.stats.steps++; + r.stats.ms_step += lap(tp); + if (head) head->commit_rows(std::vector{t1}, base.last_hidden(), 1); + if ((int)r.tokens.size() >= opt.max_tokens) break; + continue; + } + d.resize((size_t)nd); + if (opt.tau > 0.0f && conf.size() == d.size()) { + for (int i = 0; i < nd; i++) + if (conf[(size_t)i] < opt.tau) { nd = i + 1; break; } + d.resize((size_t)nd); + } + std::vector batch{t1}; + batch.insert(batch.end(), d.begin(), d.end()); + if (!base.step_batch_all(batch, pos, all)) { r.err = "verify batch failed"; return r; } + r.stats.ms_batch += lap(tp); + r.stats.iterations++; + r.stats.batches++; + r.stats.rows_verified += nd; + r.stats.drafted += nd; + for (int i = 0; i < nd && i < Stats::kMaxDepth; i++) r.stats.drafted_at[i]++; + int a = 0; + while (a < nd && argmax(&all[(size_t)a * (size_t)V], V) == d[(size_t)a]) a++; + // nd <= left, so every accepted draft fits the budget; only the consumer can stop the prefix early. + int take = 0; + bool stopped = false; + for (int i = 0; i < a; i++) { + const bool keep = emit(d[(size_t)i]); + take = i + 1; + if (!keep) { stopped = true; break; } + } + r.stats.accepted += a; + r.stats.chain_drafts.push_back(d); + r.stats.chain_accepted.push_back(a); + for (int i = 0; i < a && i < Stats::kMaxDepth; i++) r.stats.accepted_at[i]++; + for (int i = 0; i <= a && i < nd && i < Stats::kMaxDepth; i++) r.stats.reached_at[i]++; + if (a == 0) r.stats.first_reject++; + if (!base.commit_verify(take + 1)) { r.err = "commit_verify refused"; return r; } + const std::vector& rows = base.last_hidden_rows(); + const size_t h = (size_t)base.hidden_size(); + const size_t got = (h && rows.size() % h == 0) ? rows.size() / h : 0; + if (!rows.empty()) { + if (got != (size_t)(take + 1) && got != (size_t)(nd + 1)) { + r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " + + std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)"; + return r; + } + std::vector kept{t1}; + kept.insert(kept.end(), d.begin(), d.begin() + take); + head->commit_rows(kept, rows, take + 1); + } + r.stats.ms_commit += lap(tp); + logits.assign(all.begin() + (size_t)take * (size_t)V, all.begin() + (size_t)(take + 1) * (size_t)V); + pos += take + 1; + if (stopped || (int)r.tokens.size() >= opt.max_tokens) break; + continue; + } + // Consume t1 so the head sees the state after it; `have` is the position the drafts start at. if (!base.step(t1, pos, logits)) { r.err = "step failed"; return r; } r.stats.steps++; diff --git a/tests/mtp_draft_test.cpp b/tests/mtp_draft_test.cpp index 37730c6b0..9ccb5e7b9 100644 --- a/tests/mtp_draft_test.cpp +++ b/tests/mtp_draft_test.cpp @@ -26,6 +26,7 @@ #include "mtp/draft.h" #include "mtp/mtp_head.h" +#include #include #include #include @@ -288,6 +289,42 @@ class RandomHead : public DraftHead { bool same(const std::vector& a, const std::vector& b) { return a == b; } +// [17]'s head: drafts from the position alone and declares needs_last_hidden() == false, the shape of a DFlash draft +// (it reads the base's committed rows, not the hidden after the last token), so run() takes the one-pass round. It also +// follows its own stream: every committed row must reach commit_rows() exactly once and in order, t1's row included. +class OnePassHead : public DraftHead { +public: + enum Kind { kPerfect, kGarbage, kHalf }; + OnePassHead(Kind kind, int prompt_len) : kind_(kind), expect_(prompt_len + 1) {} + bool needs_last_hidden() const override { return false; } + int draft(const std::vector& hidden, int, int pos, int k, std::vector& out, + std::vector* conf = nullptr) override { + if (!hidden.empty()) handed_hidden_++; + if (conf) conf->assign((size_t)k, 1.0f); + out.clear(); + for (int i = 0; i < k; i++) { + const int t = truth(pos + i); + out.push_back(kind_ == kPerfect || (kind_ == kHalf && i == 0) ? t : (t + 1) % V); + } + return (int)out.size(); + } + void commit_rows(const std::vector& tokens, const std::vector& rows, int M) override { + if ((int)tokens.size() != M || (int)rows.size() < M) { bad_++; return; } + for (int m = 0; m < M; m++) + if ((int)rows[(size_t)m] != expect_ + m) bad_++; // the toy's row m is "positions consumed after it" + expect_ += M; + seen_ += M; + } + int seen() const { return seen_; } + int bad() const { return bad_; } + int handed_hidden() const { return handed_hidden_; } + +private: + Kind kind_; + int expect_; + int seen_ = 0, bad_ = 0, handed_hidden_ = 0; +}; + // ---- the drafter's own stream (MtpDrafter's cursor) ------------------------------------------------------------ // // The MTP head attends over its own states, so a draft is worth something only when the drafter's cache IS the base @@ -910,6 +947,93 @@ int main() { r4.stats.accepted, r8.stats.accepted, r4.stats.drafted, r8.stats.drafted); } + // [17] the one-pass round (engine#1781): a head that does not read the hidden after t1 gets [t1, d1..dk] verified as + // ONE batch. Pinned: the text and the recurrent state equal plain decode's for every accept shape, k is held to + // max_batch_rows() - 1 (t1 takes a row), a fully accepted chain costs one base pass per k + 1 tokens and no + // step, and the drafter sees every committed row once, in order. + { + ToyBase rm; + rm.use_recurrence(true); + const Result rref = onebit::npu::mtp::run_plain(rm, P, base_opt); + const char* names[] = {"perfect", "garbage", "half"}; + for (int k : {1, 2, 4, 7, 12}) { + for (int kind = 0; kind < 3; kind++) { + for (int keep : {0, 1}) { + ToyBase m(8); + m.use_recurrence(true); + m.keep_batch_rows(keep != 0); + OnePassHead h((OnePassHead::Kind)kind, (int)P.size()); + Options o = base_opt; + o.k = k; + const Result r = onebit::npu::mtp::run(m, &h, P, o); + const int kk = std::min(k, 7); + char msg[200]; + std::snprintf(msg, sizeof msg, + "one-pass k=%d head=%s rows=%s: text, state and positions equal plain decode", + k, names[kind], keep ? "batch" : "prefix"); + check(r.ok && same(r.tokens, ref.tokens) && m.state() == rm.state() && m.bad_pos() == 0 && + m.consumed() == (int)P.size() + (int)r.tokens.size() && m.pending() == 0, + msg); + std::snprintf(msg, sizeof msg, "one-pass k=%d head=%s rows=%s: never past %d rows (over=%d)", k, + names[kind], keep ? "batch" : "prefix", m.max_batch_rows(), m.over_batch()); + check(m.over_batch() == 0, msg); + std::snprintf(msg, sizeof msg, + "one-pass k=%d head=%s rows=%s: drafter saw %d rows in order (want %d), no hidden", + k, names[kind], keep ? "batch" : "prefix", h.seen(), m.consumed() - (int)P.size()); + check(r.ok && h.bad() == 0 && h.seen() == m.consumed() - (int)P.size() && h.handed_hidden() == 0, + msg); + if (kind == OnePassHead::kPerfect) { + const int want_batches = (WANT + kk) / (kk + 1); + std::snprintf(msg, sizeof msg, + "one-pass k=%d perfect rows=%s: %d batches (want %d), %d steps (want the %zu of " + "the prompt), every draft accepted", + k, keep ? "batch" : "prefix", m.batches(), want_batches, m.steps(), P.size()); + check(m.batches() == want_batches && m.steps() == (int)P.size() && + r.stats.accepted == r.stats.drafted, + msg); + } else if (kind == OnePassHead::kGarbage) { + std::snprintf(msg, sizeof msg, + "one-pass k=%d garbage rows=%s: nothing accepted, one base pass per token (%d)", k, + keep ? "batch" : "prefix", m.batches() + m.steps() - (int)P.size()); + check(r.stats.accepted == 0 && m.batches() + m.steps() - (int)P.size() == WANT, msg); + } else if (k > 1) { + std::snprintf(msg, sizeof msg, "one-pass k=%d half rows=%s: the first draft only (%d of %d)", k, + keep ? "batch" : "prefix", r.stats.accepted, r.stats.drafted); + check(r.stats.accepted == r.stats.iterations && r.stats.accepted < r.stats.drafted, msg); + } + } + } + } + // The same perfect chain through the two-pass round, for the number the one-pass round exists to change. + ToyBase m2(8), m1(8); + PerfectHead two; + OnePassHead one(OnePassHead::kPerfect, (int)P.size()); + Options o = base_opt; + o.k = 7; + const Result r2 = onebit::npu::mtp::run(m2, &two, P, o); + const Result r1 = onebit::npu::mtp::run(m1, &one, P, o); + std::printf("INFO one-pass vs two-pass, perfect head, k=7: %.3f vs %.3f base passes per token\n", + r1.stats.passes_per_token(), r2.stats.passes_per_token()); + check(r1.ok && r2.ok && same(r1.tokens, r2.tokens) && m1.steps() + m1.batches() < m2.steps() + m2.batches(), + "one-pass k=7: same text as the two-pass round in fewer base passes"); + // A consumer that stops inside an accepted prefix stops the run there, and the refused token stays committed + // (run()'s contract on both rounds). + int n = 0; + ToyBase ms(8); + OnePassHead hs(OnePassHead::kPerfect, (int)P.size()); + const Result rs = onebit::npu::mtp::run(ms, &hs, P, o, [&](int) { return ++n < 6; }); + check(rs.ok && rs.tokens.size() == 6 && same(rs.tokens, std::vector(ref.tokens.begin(), ref.tokens.begin() + 6)) && + ms.consumed() == (int)P.size() + 6 && ms.pending() == 0, + "one-pass: on_token false on the 6th token stops the run with 6 tokens emitted and committed"); + // A batch that holds one row leaves no row for a draft: plain decode, no batch. + ToyBase m0(1); + OnePassHead h0(OnePassHead::kPerfect, (int)P.size()); + const Result r0 = onebit::npu::mtp::run(m0, &h0, P, o); + check(r0.ok && same(r0.tokens, ref.tokens) && m0.batches() == 0 && m0.over_batch() == 0 && + h0.seen() == WANT, + "one-pass with max_batch_rows() == 1: plain steps, no batch, the drafter still sees every row"); + } + std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails); return fails ? 1 : 0; }