From eadff3a804fd5d9eb23b2d3d7e2d8f75cd55d370 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 03:10:44 -0400 Subject: [PATCH 1/2] serve: forward-serve --spec-lookup K, the speculative loop with a prompt-lookup drafter, token for token the plain reply (refs #1781) - npu/mtp/prompt_lookup.h: PromptLookupHead, a drafter with no model: the up-to-K tokens that followed the latest earlier occurrence of the stream's last 3..1 tokens. One-pass (needs_last_hidden() false). - npu/mtp/draft.h: the one-pass round calls commit_rows() even when the base keeps no hidden rows (a token-only drafter follows the stream through it; before, it never saw a commit); Options::consume_last (false skips running the reply's last token, as forward-serve's plain loop does). - app/forward_serve.cpp: --spec-lookup K / --spec-fallback X. A greedy request without grammar, logprobs or logit dump decodes through mtp::run_from(); every token, on either loop, goes through one deliver() (end tokens, grammar, stop sequences, hold-back). Host-GEMM verifies 16 rows. One [spec] log line per request. Tests: tests/prompt_lookup_test.cpp (the lookup by hand; through mtp::run: text = plain, stream = committed text, fewer passes on a repeating answer); mtp_draft [18] consume_last; tests/serve_spec_lookup_test.py (ctest serve_spec_lookup, ONEBIT_NPU builds): fabricated container served host-GEMM plain and with --spec-lookup 4, equal text/finish/completion_tokens over budgets, stop sequences that cut the reply and a seeded sampled request; spec path taken on exactly the greedy requests. Mutants: no greedy gate, deliver() result ignored, host rows 1, no commit_rows without rows -- each fails a test. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 10 ++ app/forward_serve.cpp | 117 +++++++++++++++----- docs/npu.md | 18 ++++ npu/mtp/draft.h | 23 ++-- npu/mtp/prompt_lookup.h | 78 ++++++++++++++ tests/mtp_draft_test.cpp | 19 ++++ tests/prompt_lookup_test.cpp | 147 +++++++++++++++++++++++++ tests/serve_spec_lookup_test.py | 183 ++++++++++++++++++++++++++++++++ 8 files changed, 562 insertions(+), 33 deletions(-) create mode 100644 npu/mtp/prompt_lookup.h create mode 100644 tests/prompt_lookup_test.cpp create mode 100644 tests/serve_spec_lookup_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 100c69d37..6384298b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -594,6 +594,11 @@ if(TARGET OpenMP::OpenMP_CXX) target_link_libraries(mtp_draft_test PRIVATE OpenMP::OpenMP_CXX) endif() add_test(NAME mtp_draft COMMAND mtp_draft_test) +# engine#1781: the prompt-lookup drafter (npu/mtp/prompt_lookup.h), by hand and through mtp::run() on a toy base. +add_executable(prompt_lookup_test tests/prompt_lookup_test.cpp) +target_include_directories(prompt_lookup_test PRIVATE npu) +add_dependencies(onebit prompt_lookup_test) +add_test(NAME prompt_lookup COMMAND prompt_lookup_test) # engine#1781 gate 2: the distribution-following acceptance rule (accept with min(1, p/q), resample the residual at the # first rejection), with the temperature-0 identity and a deliberately wrong drafter whose output must still follow p. # Header-only and device-free; pulled into CI through npu_sampling_test for the same reason as mtp_draft_test below. @@ -2111,6 +2116,11 @@ endif() set_tests_properties(gdn_batch_order PROPERTIES SKIP_RETURN_CODE 77) set_tests_properties(layer_taps_model PROPERTIES SKIP_RETURN_CODE 77) set_tests_properties(commit_full_accept PROPERTIES SKIP_RETURN_CODE 77) + # engine#1781: forward-serve --spec-lookup gives the plain loop's replies, end to end, host-GEMM on a container the + # script fabricates (no device, no model on disk). forward-serve exists only in this build, so it is a dev-box row. + add_test(NAME serve_spec_lookup + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/serve_spec_lookup_test.py $) + set_tests_properties(serve_spec_lookup PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600) set_tests_properties(batch_check_state PROPERTIES SKIP_RETURN_CODE 77) # The fast-lane end-to-end test (npu_lane_e2e) is retired: its kernels were FastFlowLM-derived # and its reference logits are lost. The NPU check is the private dx add-on's from-source diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 209f003ba..87598cb93 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -54,6 +54,7 @@ #include "npu_systemone.h" #include "tokenizer.h" #include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653) +#include "../npu/mtp/prompt_lookup.h" // --spec-lookup: the speculative loop with a prompt-lookup drafter (#1781) #include "rt.h" // the own NPU runtime (ONEBIT_NPU_RT_DIR) #include @@ -139,6 +140,12 @@ struct ForwardEngine { // KL-divergence comparison against another stack (engine#671 gate b) needs the distribution // itself. Empty means off, which is the default: nothing is written unless asked. std::string dump_logits; + // --spec-lookup K (engine#1781): a greedy request without grammar or logprobs decodes through mtp::run_from() with + // the prompt-lookup drafter, up to K drafts verified per batch; 0 = off (the default). --spec-fallback X: stop + // drafting for the rest of a reply once its last 3 rounds averaged fewer than X tokens (0 = never). + int spec_lookup = 0; + double spec_fallback = 0.0; + bool host_gemm = false; // NPU_INFER_HOST_GEMM: the forward runs on the host, with no launch width of its own // The dump's response index. `step_.bin` used the decode index alone, so a second request overwrote the first // request's steps: a multi-prompt comparison (the KLD row, engine#803) then silently compared only the LAST prompt's // dumps and looked complete. Numbering the response too makes the set of files say how many responses it holds. @@ -529,28 +536,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib }; bool stopped_at_eos = false, stopped_at_stop = false; int produced = 0; - for (int n = 0; n < limit; n++) { - // the client is gone: free the NPU (the bridge knows first when a write to it failed) - if (req.is_connection_closed() || (bridge && bridge->cancelled())) break; - if (gram) refresh_allowed(); - // Raw distribution for a reference comparison: sample_token() takes logits by const - // reference and the copy below happens before it runs, so what lands on disk is what - // the model produced, not what the sampler chose. - if (!e.dump_logits.empty()) { - const std::string path = e.dump_logits + "/step_" + std::to_string(dump_req) + "_" + - std::to_string(n) + ".bin"; - if (std::FILE* f = onebit_fopen_private_wb(path.c_str())) { // 0600, no symlink (file_create.h) - std::fwrite(logits.data(), sizeof(float), logits.size(), f); - std::fclose(f); - } - } - const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size()); - // -1: the grammar allows no token here, so the answer ends where it is. An end token is - // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply - // that completes reports "stop" instead of padding to the budget (engine#1182). - if (next < 0) { stopped_at_eos = true; break; } + // One chosen token through the reply: the end tokens, the grammar, the stop sequences and the hold-back that + // keeps a partial stop match unsent. false = the reply ends here (an end token, a stop sequence, or a client + // that stopped reading). The plain loop and the speculative one hand every token to this, in order. + auto deliver = [&](int next) -> bool { if (std::find(e.eos.begin(), e.eos.end(), next) != e.eos.end() || - (e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; break; } + (e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; return false; } if (gram) gram->feed(grammar_state, e.tok_text[next]); const std::string piece = e.tok->decode(next); pending += piece; @@ -564,7 +555,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (stop_at != std::string::npos) { if (stop_at > 0) on_text(pending.substr(0, stop_at), take_held(stop_at)); stopped_at_stop = true; - break; + return false; } // Send what no stop sequence can still match, and keep the tail: the next // token may complete a match that starts in it. @@ -575,8 +566,75 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (emit > 0) { const std::string out = pending.substr(0, emit); pending.erase(0, emit); - if (!on_text(out, take_held(emit))) break; + if (!on_text(out, take_held(emit))) return false; + } + return true; + }; + // engine#1781: the speculative path, only where it cannot change a token. Greedy, so each token is the bounded + // argmax the verify rows check drafts against; no grammar (its mask is per token and per state); no logprobs + // and no logit dump (both read the logits of every position, and a verify batch's rows are not kept). The text + // is then the plain loop's token for token -- what tests/serve_spec_lookup_test.py holds it to. + const bool spec = e.spec_lookup > 0 && opts.sample.greedy() && !gram && !want_lp && e.dump_logits.empty(); + if (spec) { + struct ServeBase : npu::mtp::BaseModel { + Q4nxNpuForward& f; + int nv, rows; + ServeBase(Q4nxNpuForward& fw, int v, int r) : f(fw), nv(v), rows(r) {} + int vocab() const override { return nv; } + int hidden_size() const override { return int(f.last_hidden().size()); } + int max_batch_rows() const override { return rows; } + bool step(int t, int p, std::vector& l) override { return f.step(t, p, l); } + bool step_batch_all(const std::vector& t, int s0, std::vector& l) override { + f.begin_verify(int(t.size()), s0); + return f.step_batch_all(t, s0, l); + } + bool commit_verify(int a) override { return f.commit_verify_rows(a); } + const std::vector& last_hidden() const override { return f.last_hidden(); } + }; + // The bound sample_token() applies: ids past the tokenizer are padding and never chosen (engine#2017). + // The device verifies as many rows as its kernels are wide; host GEMMs have no width, and 1 (what the + // forward reports there) would leave no row for a draft, so host mode verifies the device's 16. + ServeBase base(*e.fwd, int(npu::token_bound(logits.size(), e.tok->size())), + e.host_gemm ? 16 : e.fwd->max_batch_rows()); + npu::mtp::PromptLookupHead head(ids); + npu::mtp::Options o; + o.k = e.spec_lookup; + o.max_tokens = limit; + o.tau = 0.0f; + o.fallback_below = e.spec_fallback; + o.consume_last = false; // the plain loop does not run the reply's last token either + const npu::mtp::Result r = npu::mtp::run_from( + base, &head, int(ids.size()), logits, o, [&](int t) { + if (req.is_connection_closed() || (bridge && bridge->cancelled())) return false; + return deliver(t); + }); + if (!r.ok) throw std::runtime_error("speculative decode: " + r.err); + std::fprintf(stderr, "[spec] %d tokens: %d rounds, %d of %d drafts accepted, %d plain steps%s\n", + int(r.tokens.size()), r.stats.batches, r.stats.accepted, r.stats.drafted, r.stats.steps, + r.stats.fell_back_at >= 0 ? (", fell back after round " + std::to_string(r.stats.fell_back_at)).c_str() + : ""); + } + for (int n = 0; !spec && n < limit; n++) { + // the client is gone: free the NPU (the bridge knows first when a write to it failed) + if (req.is_connection_closed() || (bridge && bridge->cancelled())) break; + if (gram) refresh_allowed(); + // Raw distribution for a reference comparison: sample_token() takes logits by const + // reference and the copy below happens before it runs, so what lands on disk is what + // the model produced, not what the sampler chose. + if (!e.dump_logits.empty()) { + const std::string path = e.dump_logits + "/step_" + std::to_string(dump_req) + "_" + + std::to_string(n) + ".bin"; + if (std::FILE* f = onebit_fopen_private_wb(path.c_str())) { // 0600, no symlink (file_create.h) + std::fwrite(logits.data(), sizeof(float), logits.size(), f); + std::fclose(f); + } } + const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size()); + // -1: the grammar allows no token here, so the answer ends where it is. An end token is + // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply + // that completes reports "stop" instead of padding to the budget (engine#1182). + if (next < 0) { stopped_at_eos = true; break; } + if (!deliver(next)) break; // No further token will be sampled after this one (limit reached), so the // decode step that would only produce the NEXT position's logits is a whole // weight stream -- 8.7 s on the 27B -- spent on a result nobody reads. At @@ -1062,6 +1120,8 @@ int run_forward_serve(int argc, char** argv) { npu::PrefixOptions prefix_opts; // --prefix-slots / --prefix-cache-mb / --prefix-min-gain (#650) bool prefix_pin = false; // --prefix-pin (engine#1785) int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's) + int spec_lookup = 0; // --spec-lookup K (engine#1781): prompt-lookup drafts, K per verify batch + double spec_fallback = 0.0; // --spec-fallback X: stop drafting below X tokens per round for (int i = 0; i < argc; i++) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -1081,10 +1141,16 @@ int run_forward_serve(int argc, char** argv) { else if (a == "--npu-streams") streams = std::stoi(next()); else if (a == "--npu-target") npu_target = next(); else if (a == "--dump-logits") dump_logits = next(); + else if (a == "--spec-lookup") spec_lookup = std::stoi(next()); + else if (a == "--spec-fallback") spec_fallback = std::stod(next()); else throw std::runtime_error("unknown option " + a); } if (model.empty()) - throw std::runtime_error("usage: 1bit forward-serve -m --kernels [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits ]"); + throw std::runtime_error("usage: 1bit forward-serve -m --kernels [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits ] [--spec-lookup K [--spec-fallback X]]"); + if (spec_lookup < 0 || spec_lookup > 15) + throw std::runtime_error("--spec-lookup takes 0..15 drafts per round (a verify batch holds at most 16 rows), not " + + std::to_string(spec_lookup)); + if (!(spec_fallback >= 0.0)) throw std::runtime_error("--spec-fallback takes a number of tokens >= 0"); if (streams < 1 || streams > kNpuMaxStreams) throw std::runtime_error("--npu-streams takes 1.." + std::to_string(kNpuMaxStreams) + " (the device's hwctx limit), not " + std::to_string(streams)); @@ -1104,6 +1170,8 @@ int run_forward_serve(int argc, char** argv) { ForwardEngine e; e.id = alias.empty() ? std::filesystem::path(dir).filename().string() : alias; + e.spec_lookup = spec_lookup; + e.spec_fallback = spec_fallback; if (!dump_logits.empty()) { std::error_code ec; std::filesystem::create_directories(dump_logits, ec); @@ -1261,6 +1329,7 @@ int run_forward_serve(int argc, char** argv) { // other lanes' device work. Open the device, and the lock, only when the forward is // going to use it. const bool host_gemm = std::getenv("NPU_INFER_HOST_GEMM") != nullptr; + e.host_gemm = host_gemm; if (npu_lock_taken(!host_gemm, std::getenv("NPU_NO_DEVICE_LOCK") != nullptr)) e.lock_fd = take_npu_device_lock(streams); std::unique_ptr dev; diff --git a/docs/npu.md b/docs/npu.md index 99475c43e..f520cf8a5 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3421,6 +3421,24 @@ fewer than `fallback_below` tokens on average, the run finishes with plain steps round). `tools/dflash2_probe loop` with `DFLASH2_MTP_RUN=1` drives the served 27B through exactly these pieces, with `DFlashHead` and the DFlash 2 selector as its pick hook, after the same batched prefill as its hand-written loop. +**Serving it: `forward-serve --spec-lookup K`** (off by default; `--spec-fallback X` adds the fallback). A request that +is greedy, has no grammar, and asks for no logprobs (and a server without `--dump-logits`) decodes through +`mtp::run_from()` with the prompt-lookup drafter (`npu/mtp/prompt_lookup.h`). That drafter needs no model: it proposes +the up to `K` tokens that followed the latest earlier occurrence of the stream's last one to three tokens. Every +other request takes the plain loop. The rule is that speculation is used only where it cannot change a token: each +token is still the tokenizer-bounded argmax, and both loops hand every token to the same code for end tokens, stop +sequences and the stop hold-back. The reply's last token is not run through the model (`Options::consume_last = +false`), as in the plain loop. The server logs one `[spec]` line per request (rounds, drafts accepted, plain steps). +`tests/serve_spec_lookup_test.py` (ctest `serve_spec_lookup`, on a build with `ONEBIT_NPU`) holds it to that end to +end. It fabricates a container, serves it host-GEMM twice (plain, then with `--spec-lookup 4`), and requires the same +text, `finish_reason` and `completion_tokens` for: +- every budget; +- stop sequences cut from the plain replies, so they do end them; +- a seeded sampled request, which must stay on the plain loop. + +In host-GEMM mode the verify batch is 16 rows, the device's width. Host GEMMs have no width of their own, and the 1 +the forward reports there would leave no row for a draft. The device route has not been measured with it yet. + **What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth 6 measured 0 %). diff --git a/npu/mtp/draft.h b/npu/mtp/draft.h index df2fd4f89..3f1546043 100644 --- a/npu/mtp/draft.h +++ b/npu/mtp/draft.h @@ -173,6 +173,10 @@ struct Options { // fallback_below <= 0 turns it off. double fallback_below = 0.0; int fallback_window = 3; + // When the budget ends on the base's own token, whether that token is still run through the base. true keeps the + // base holding every emitted token (what the tests and a caller that continues the stream want); false skips a + // pass whose logits nobody reads, which is what forward-serve does at the end of a reply. + bool consume_last = true; }; struct Stats { @@ -277,6 +281,7 @@ inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector= opt.max_tokens) break; if (k <= 0 || fallback.off()) { // plain decode (no drafter, k = 0, or the fallback stopped it) if (!base.step(t1, pos++, logits)) { r.err = "step failed"; return r; } @@ -346,16 +351,16 @@ inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector& rows = base.last_hidden_rows(); const size_t h = (size_t)base.hidden_size(); const size_t got = (h && rows.size() % h == 0) ? rows.size() / h : 0; - if (!rows.empty()) { - if (got != (size_t)(take + 1) && got != (size_t)(nd + 1)) { - r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " + - std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)"; - return r; - } - std::vector kept{t1}; - kept.insert(kept.end(), d.begin(), d.begin() + take); - head->commit_rows(kept, rows, take + 1); + if (!rows.empty() && got != (size_t)(take + 1) && got != (size_t)(nd + 1)) { + r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " + + std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)"; + return r; } + // Always, even when the base keeps no rows (`rows` is then empty): a drafter that reads only the tokens + // (the prompt-lookup head) follows the stream through these calls alone. + std::vector kept{t1}; + kept.insert(kept.end(), d.begin(), d.begin() + take); + head->commit_rows(kept, rows, take + 1); r.stats.ms_commit += lap(tp); logits.assign(all.begin() + (size_t)take * (size_t)V, all.begin() + (size_t)(take + 1) * (size_t)V); pos += take + 1; diff --git a/npu/mtp/prompt_lookup.h b/npu/mtp/prompt_lookup.h new file mode 100644 index 000000000..2d07a7d18 --- /dev/null +++ b/npu/mtp/prompt_lookup.h @@ -0,0 +1,78 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// prompt_lookup.h -- a drafter with no model: propose what followed the latest earlier occurrence of the stream's +// last tokens (engine#1781). +// +// The stream is the prompt plus every committed token, the last emitted token (t1) included. To draft, take its last n +// tokens (n from max_ngram down to min_ngram, the first length that matches wins), find their latest earlier +// occurrence, and propose the tokens that followed it. A reply that repeats its prompt -- code being edited, a quoted +// passage, a list that follows a pattern -- is then drafted at no cost but the verify batch, and a reply that does not +// gets no proposal at all, so the round falls back to a plain step. It never reads the base's hidden, so mtp::run() +// takes the one-pass round (needs_last_hidden() is false), and mtp::run's fallback bounds what a run of misses costs. +#pragma once + +#include +#include + +#include "draft.h" + +namespace onebit::npu::mtp { + +class PromptLookupHead : public DraftHead { +public: + explicit PromptLookupHead(std::vector prompt, int max_ngram = 3, int min_ngram = 1) + : prompt_(std::move(prompt)), stream_(prompt_), max_n_(max_ngram), min_n_(min_ngram < 1 ? 1 : min_ngram) {} + + bool needs_last_hidden() const override { return false; } + + int draft(const std::vector& /*hidden*/, int last_token, int pos, int k, std::vector& out, + std::vector* conf = nullptr) override { + out.clear(); + if (conf) conf->clear(); // no probabilities: mtp::run skips its tau gate + // last_token runs at pos - 1, right after the committed stream; anything else means the stream is not the + // base's, and a proposal from it would be a guess about some other text. + if (k <= 0 || pos != int(stream_.size()) + 1) return 0; + stream_.push_back(last_token); + const int len = int(stream_.size()); + for (int n = std::min(max_n_, len - 1); n >= min_n_; n--) { + const int* suf = stream_.data() + (len - n); + for (int i = len - n - 1; i >= 0; i--) { // latest earlier occurrence first + bool hit = true; + for (int j = 0; j < n && hit; j++) hit = stream_[size_t(i + j)] == suf[j]; + if (!hit) continue; + for (int j = i + n; j < len && int(out.size()) < k; j++) out.push_back(stream_[size_t(j)]); + break; + } + if (!out.empty()) break; + } + stream_.pop_back(); + return int(out.size()); + } + + void commit_rows(const std::vector& tokens, const std::vector& /*rows*/, int M) override { + for (int i = 0; i < M && i < int(tokens.size()); i++) stream_.push_back(tokens[size_t(i)]); + } + + void reset() override { stream_ = prompt_; } + + const std::vector& stream() const { return stream_; } + +private: + std::vector prompt_, stream_; + int max_n_, min_n_; +}; + +} // namespace onebit::npu::mtp diff --git a/tests/mtp_draft_test.cpp b/tests/mtp_draft_test.cpp index dc6dd6a6f..e98685d8b 100644 --- a/tests/mtp_draft_test.cpp +++ b/tests/mtp_draft_test.cpp @@ -1087,6 +1087,25 @@ int main() { const Result re = onebit::npu::mtp::run(eq, &oh, P, f); check(re.ok && same(re.tokens, ref.tokens) && re.stats.fell_back_at == -1, "fallback: rounds of exactly 2 tokens against a bar of 2.0 keep drafting"); + // consume_last = false: a run whose budget ends on the base's own token does not pay the pass after it. + for (int kind : {0, 1}) { + ToyBase m1(8), m2(8); + OnePassHead h1((OnePassHead::Kind)kind, (int)P.size()), h2((OnePassHead::Kind)kind, (int)P.size()); + Options c = base_opt; + c.k = 4; + const Result a1 = onebit::npu::mtp::run(m1, &h1, P, c); + c.consume_last = false; + const Result a2 = onebit::npu::mtp::run(m2, &h2, P, c); + const int passes1 = m1.steps() + m1.batches(), passes2 = m2.steps() + m2.batches(); + char msg[200]; + std::snprintf(msg, sizeof msg, "consume_last=false, %s head: same text, %d passes against %d, base at %d of %d", + kind == 0 ? "perfect" : "garbage", passes2, passes1, m2.consumed(), m1.consumed()); + // perfect at k=4 ends inside a batch (nothing to skip); garbage ends on the base's own token (one skipped) + const int skipped = kind == 0 ? 0 : 1; + check(a1.ok && a2.ok && same(a1.tokens, a2.tokens) && passes2 == passes1 - skipped && + m2.consumed() == m1.consumed() - skipped, + msg); + } } std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails); diff --git a/tests/prompt_lookup_test.cpp b/tests/prompt_lookup_test.cpp new file mode 100644 index 000000000..962b5ceb6 --- /dev/null +++ b/tests/prompt_lookup_test.cpp @@ -0,0 +1,147 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// prompt_lookup_test.cpp -- the prompt-lookup drafter (npu/mtp/prompt_lookup.h, engine#1781). Host only, no model. +// [1] the lookup itself, by hand: the longest suffix that matches wins, its latest earlier occurrence wins, the +// continuation runs up to k tokens and may run into the suffix, nothing matches -> no proposal, a position the +// stream does not end at -> no proposal; +// [2] through mtp::run() on a toy base: the text is plain decode's whatever the base does; on a base whose answer +// repeats, drafts are accepted and the run takes fewer passes than plain decode; the head's stream is the +// committed text. + +#include "mtp/prompt_lookup.h" + +#include +#include +#include + +using onebit::npu::mtp::BaseModel; +using onebit::npu::mtp::Options; +using onebit::npu::mtp::PromptLookupHead; +using onebit::npu::mtp::Result; + +namespace { + +int fails = 0; +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", what.c_str()); + if (!ok) fails++; +} + +std::vector draft(PromptLookupHead& h, int last, int k) { + std::vector out; + h.draft({}, last, int(h.stream().size()) + 1, k, out); + return out; +} + +constexpr int V = 64; + +// next = rule(token, pos): `period` > 0 makes the answer a loop of that period, 0 makes every position different. +class Base : public BaseModel { +public: + explicit Base(int period) : period_(period) {} + int vocab() const override { return V; } + int hidden_size() const override { return 1; } + int max_batch_rows() const override { return 8; } + int next(int token, int pos) const { return period_ > 0 ? (token + 1) % period_ + 10 : (pos * 7 + 3) % V; } + bool step(int t, int p, std::vector& l) override { + if (p != consumed_) return false; + consumed_++; + steps_++; + l.assign(V, 0.0f); + l[size_t(next(t, p))] = 1.0f; + return true; + } + bool step_batch_all(const std::vector& t, int s, std::vector& all) override { + if (s != consumed_ || int(t.size()) > 8) return false; + all.assign(t.size() * size_t(V), 0.0f); + for (size_t m = 0; m < t.size(); m++) all[m * size_t(V) + size_t(next(t[m], s + int(m)))] = 1.0f; + pending_ = int(t.size()); + consumed_ += pending_; + batches_++; + return true; + } + bool commit_verify(int a) override { + if (a < 0 || a > pending_) return false; + consumed_ -= pending_ - a; + pending_ = 0; + return true; + } + const std::vector& last_hidden() const override { return h_; } + int consumed() const { return consumed_; } + int passes() const { return steps_ + batches_; } + +private: + int period_, consumed_ = 0, pending_ = 0, steps_ = 0, batches_ = 0; + std::vector h_{0.0f}; +}; + +} // namespace + +int main() { + { + PromptLookupHead h({1, 2, 3, 4, 5, 1, 2}); + check(draft(h, 3, 4) == std::vector({4, 5, 1, 2}), "suffix 1 2 3 found at the start: 4 5 1 2 follow"); + check(draft(h, 3, 9) == std::vector({4, 5, 1, 2, 3}), "k past the stream: the continuation stops at its end"); + } + { + PromptLookupHead h({7, 8, 1, 7, 8, 2, 7}); + check(draft(h, 8, 3) == std::vector({2, 7, 8}), "two earlier 7 8: the latest one's continuation (2 7 8)"); + } + { + PromptLookupHead h({3, 9, 6, 1, 9, 7, 3}); + check(draft(h, 9, 3) == std::vector({6, 1, 9}), + "3 9 matches at 0 (6 1 9), 9 alone last at 4 (7 3 9): the longer suffix wins"); + } + { + PromptLookupHead h({1, 2, 3}); + check(draft(h, 4, 4).empty(), "a token never seen before: no proposal"); + std::vector out; + check(h.draft({}, 3, 7, 4, out) == 0 && out.empty(), "a position the stream does not end at: no proposal"); + check(h.stream() == std::vector({1, 2, 3}), "drafting leaves the stream as it was"); + h.commit_rows({4, 5}, {}, 2); + check(h.stream() == std::vector({1, 2, 3, 4, 5}), "commit_rows appends the committed tokens"); + h.reset(); + check(h.stream() == std::vector({1, 2, 3}), "reset goes back to the prompt"); + } + const std::vector prompt = {10, 11, 12, 13, 14, 15, 10, 11}; + for (int period : {6, 0}) { + for (int k : {1, 3, 7}) { + Options o; + o.max_tokens = 40; + o.k = k; + o.tau = 0.0f; + Base pb(period), b(period); + const Result plain = onebit::npu::mtp::run_plain(pb, prompt, o); + PromptLookupHead h(prompt); + const Result r = onebit::npu::mtp::run(b, &h, prompt, o); + std::vector stream = prompt; + stream.insert(stream.end(), r.tokens.begin(), r.tokens.end()); + const std::string tag = "period " + std::to_string(period) + " k=" + std::to_string(k) + ": "; + check(r.ok && r.tokens == plain.tokens && b.consumed() == int(prompt.size()) + 40, + tag + "text and positions are plain decode's"); + check(h.stream() == stream, tag + "the head's stream is the committed text"); + if (period > 0) + check(r.stats.accepted > 0 && b.passes() < pb.passes(), + tag + "a repeating answer: " + std::to_string(r.stats.accepted) + " drafts accepted, " + + std::to_string(b.passes()) + " passes against plain's " + std::to_string(pb.passes())); + else + std::printf("INFO %saccepted %d of %d drafts, %d passes against plain's %d\n", tag.c_str(), + r.stats.accepted, r.stats.drafted, b.passes(), pb.passes()); + } + } + std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails); + return fails ? 1 : 0; +} diff --git a/tests/serve_spec_lookup_test.py b/tests/serve_spec_lookup_test.py new file mode 100644 index 000000000..4a92fcd94 --- /dev/null +++ b/tests/serve_spec_lookup_test.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""forward-serve --spec-lookup returns the plain loop's reply, token for token (engine#1781). + +usage: tests/serve_spec_lookup_test.py <1bit binary> (needs numpy) + +The speculative path (app/forward_serve.cpp, `spec`) decodes a greedy request through mtp::run_from() with the +prompt-lookup drafter. It is only allowed where it cannot change a token, and this holds it to that end to end: the +two-layer qwen3 container tests/batch_check_state_ci.py fabricates, a 256-token byte-level tokenizer written here, and +`1bit forward-serve` in host-GEMM mode (NPU_INFER_HOST_GEMM=1: no device is opened, no lock is taken) started twice, +plain and with --spec-lookup 4. The same id prompts go to both: several budgets, stop sequences cut out of the plain +replies (so they do end their replies), and one seeded sampled request. Text, finish_reason and completion_tokens +must be equal; the spec server must take the speculative path on exactly the greedy requests and draft (its `[spec]` +lines). + +A dev-box row: forward-serve is built only with ONEBIT_NPU. Exits 77 (skip) without numpy. +""" +import importlib.util +import json +import os +import socket +import subprocess +import sys +import tempfile +import time +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.path.join(HERE, "..", "tools", "safetensors_to_q4nx.py") + + +def bytes_to_unicode(): + bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + cs = bs[:] + n = 0 + for b in range(256): + if b not in bs: + bs.append(b) + cs.append(256 + n) + n += 1 + return {b: chr(c) for b, c in zip(bs, cs)} + + +def write_tokenizer(d): + m = bytes_to_unicode() + vocab = {m[b]: b for b in range(256)} + tok = {"version": "1.0", "added_tokens": [], + "pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": False, "trim_offsets": True, "use_regex": True}, + "decoder": {"type": "ByteLevel"}, + "model": {"type": "BPE", "vocab": vocab, "merges": []}} + with open(os.path.join(d, "tokenizer.json"), "w") as f: + json.dump(tok, f) + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def post(port, body): + req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=120) as r: + return json.load(r) + + +def start(binary, model, port, extra, log): + env = dict(os.environ, NPU_INFER_HOST_GEMM="1", OMP_WAIT_POLICY="PASSIVE") + p = subprocess.Popen([binary, "forward-serve", "-m", os.path.join(model, "model.q4nx"), "--kernels", + os.path.join(model, "kernels"), "-p", str(port)] + extra, + env=env, stdout=log, stderr=subprocess.STDOUT) + for _ in range(600): + if p.poll() is not None: + return p + try: + urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1).read() + return p + except Exception: + time.sleep(0.1) + return p + + +def main(): + if len(sys.argv) != 2: + print(__doc__.strip().splitlines()[2]) + return 2 + try: + import numpy as np + except ImportError: + print("SKIP: numpy is not installed") + return 77 + spec = importlib.util.spec_from_file_location("bcs", os.path.join(HERE, "batch_check_state_ci.py")) + bcs = importlib.util.module_from_spec(spec) + spec.loader.exec_module(bcs) + fails = 0 + with tempfile.TemporaryDirectory(prefix="serve_spec_lookup.", dir=bcs.tmp_root()) as tmp: + src, out = os.path.join(tmp, "hf"), os.path.join(tmp, "q4nx") + bcs.make_model(src, np, "qwen3") + r = subprocess.run([sys.executable, TOOL, "pack", src, out], capture_output=True, text=True) + if r.returncode != 0: + print(f"FAIL packing the fabricated container: {r.stderr[-600:]}") + return 1 + write_tokenizer(out) + prompts = [[40, 41, 42, 43, 40, 41, 42, 43, 40, 41], list(range(60, 90)), [7, 7, 7, 7, 7, 7], [200, 3, 200, 3, 200]] + base = [{"prompt": p, "max_tokens": n, "temperature": 0} for p in prompts for n in (1, 5, 24)] + # A sampled request (seeded, so both servers draw the same tokens) must stay on the plain loop. + sampled = [{"prompt": prompts[1], "max_tokens": 16, "temperature": 0.8, "seed": 7}] + replies, logs, cases = {}, {}, [] + for name, extra in (("plain", []), ("spec", ["--spec-lookup", "4"])): + port = free_port() + logp = os.path.join(tmp, name + ".log") + with open(logp, "w") as log: + p = start(sys.argv[1], out, port, extra, log) + try: + if p.poll() is not None: + print(f"FAIL {name} server exited: " + open(logp).read()[-800:]) + return 1 + if name == "plain": + got = [post(port, b) for b in base] + # Stop sequences that occur: a piece of the plain reply itself, taken a few characters in, so + # each one cuts its reply mid-way (finish "stop") and the hold-back runs on both paths. + stops = [] + for j, b in zip(got, base): + t = j["choices"][0]["text"] + # printable ASCII only: a replacement character (a split UTF-8 byte) never matches + at = next((i for i in range(3, len(t) - 1) if t[i:i + 2].isascii() and t[i:i + 2].isprintable()), -1) + if b["max_tokens"] == 24 and at >= 0: + stops.append({"prompt": b["prompt"], "max_tokens": 24, "temperature": 0, + "stop": [t[at:at + 2]]}) + cases = base + stops + sampled + replies[name] = [] + for b in cases: + j = post(port, b) + c = j["choices"][0] + replies[name].append((c["text"], c["finish_reason"], j["usage"]["completion_tokens"])) + finally: + p.terminate() + try: + p.wait(timeout=20) + except subprocess.TimeoutExpired: + p.kill() + p.wait() + logs[name] = open(logp).read() + for b, a, c in zip(cases, replies["plain"], replies["spec"]): + ok = a == c + fails += not ok + what = (f"prompt {prompts.index(b['prompt'])} max_tokens {b['max_tokens']}" + + (f" stop {b['stop']!r}" if "stop" in b else "") + + (f" temperature {b['temperature']}" if b["temperature"] else "")) + print(f"{'PASS' if ok else 'FAIL'} {what}: {c[2]} tokens, finish {c[1]}" + + ("" if ok else f" (plain {a!r}, spec {c!r})")) + n_stop = sum(1 for b, c in zip(cases, replies["spec"]) if "stop" in b and c[1] == "stop") + ok = n_stop >= 2 + fails += not ok + print(f"{'PASS' if ok else 'FAIL'} {n_stop} stop-sequence requests ended on their stop sequence") + spec_lines = [l for l in logs["spec"].splitlines() if l.startswith("[spec]")] + drafted = sum(int(l.split(" of ")[1].split()[0]) for l in spec_lines) + accepted = sum(int(l.split(" drafts accepted")[0].split(", ")[-1].split(" of ")[0]) for l in spec_lines) + greedy = sum(1 for b in cases if not b["temperature"]) + ok = len(spec_lines) == greedy and drafted > 0 and "[spec]" not in logs["plain"] + fails += not ok + print(f"{'PASS' if ok else 'FAIL'} the spec server took the speculative path on {len(spec_lines)} requests, the " + f"{greedy} greedy ones ({accepted} of {drafted} drafts accepted); the sampled one and the plain server never") + print("ALL PASS" if not fails else f"FAILED {fails}") + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) From d9cdfbb6d74fa11780997647303f12a56ac7fd0e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 03:30:33 -0400 Subject: [PATCH 2/2] serve: forward-serve --spec-draft , a DFlash draft package as the speculative drafter (refs #1781) - app/forward_serve.cpp: --spec-draft [--spec-draft-k K]. The package is loaded, decoded and checked against the target before ready (width, target layers); the forward's tap sink is emptied when a request takes the engine; DFlashHead drafts with the target's head (DFlash 2's selector when the package has one, else the bounded argmax). --spec-draft and --spec-lookup are refused together. - npu/dflash_head.h: dflash2_select(), the DFlash 2 candidate selector as one function. - 1bit links npu/dflash_forward.cpp and npu/dflash_draft.cpp. tests/serve_spec_lookup_test.py: a third server, --spec-draft on a generation-1 package the script writes (tests/dflash_draft_test.cpp's synthetic layout, sized to the fabricated target): all 16 replies equal the plain server's, the speculative path on exactly the 15 greedy requests, every one of two or more tokens drafted. Mutants (sink not emptied per request, taps never set, a pick that never proposes) each fail it. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 2 + app/forward_serve.cpp | 76 +++++++++++++++-- docs/npu.md | 11 +++ npu/dflash_head.h | 29 +++++++ tests/serve_spec_lookup_test.py | 143 ++++++++++++++++++++++++++++---- 5 files changed, 238 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6384298b9..652808e79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2029,6 +2029,8 @@ endif() # same sources against the runtime stub (engine#1946) cannot drift from this one. onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) target_sources(onebit PRIVATE app/forward_serve.cpp) + # engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter. + target_sources(onebit PRIVATE npu/dflash_forward.cpp npu/dflash_draft.cpp) # engine#2111: the usage object's pure function, its own TU so tests/serve_usage_test.cpp can link it without # dragging in npu/rt.h (see cmake/tests-serve-usage.cmake). target_sources(onebit PRIVATE app/serve_usage.cpp) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 87598cb93..5fe87824d 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -55,6 +55,7 @@ #include "tokenizer.h" #include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653) #include "../npu/mtp/prompt_lookup.h" // --spec-lookup: the speculative loop with a prompt-lookup drafter (#1781) +#include "../npu/dflash_head.h" // --spec-draft: the same loop with a DFlash draft package (#1781) #include "rt.h" // the own NPU runtime (ONEBIT_NPU_RT_DIR) #include @@ -146,6 +147,15 @@ struct ForwardEngine { int spec_lookup = 0; double spec_fallback = 0.0; bool host_gemm = false; // NPU_INFER_HOST_GEMM: the forward runs on the host, with no launch width of its own + // --spec-draft (engine#1781): a DFlash draft package as the drafter instead of the prompt lookup, up to + // spec_draft_k drafts a round. The forward writes the draft's tap layers into draft_taps on every pass; the sink + // is emptied at the start of each request, so a request that resumed from a saved prefix has no rows for the + // positions it did not run, and its head refuses to draft (it decodes with plain steps, the same text). + std::unique_ptr draft; + npu::dflash::DraftWeights draft_w; + std::vector draft_hproj; // generation 2: the selector's hidden projection, decoded once + std::vector draft_taps; + int spec_draft_k = 0; // The dump's response index. `step_.bin` used the decode index alone, so a second request overwrote the first // request's steps: a multi-prompt comparison (the KLD row, engine#803) then silently compared only the LAST prompt's // dumps and looked complete. Numbering the response too makes the set of files say how many responses it holds. @@ -353,6 +363,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (room < 1) throw ContextOverflow(ids.size(), e.max_context); const int limit = std::clamp(max_tokens, 0, room); std::lock_guard lock(e.mu); + e.draft_taps.clear(); // --spec-draft: this request's tap rows start at its first position run const auto t_request = std::chrono::steady_clock::now(); std::vector logits; @@ -574,7 +585,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // argmax the verify rows check drafts against; no grammar (its mask is per token and per state); no logprobs // and no logit dump (both read the logits of every position, and a verify batch's rows are not kept). The text // is then the plain loop's token for token -- what tests/serve_spec_lookup_test.py holds it to. - const bool spec = e.spec_lookup > 0 && opts.sample.greedy() && !gram && !want_lp && e.dump_logits.empty(); + const bool spec = + (e.spec_lookup > 0 || e.draft) && opts.sample.greedy() && !gram && !want_lp && e.dump_logits.empty(); if (spec) { struct ServeBase : npu::mtp::BaseModel { Q4nxNpuForward& f; @@ -596,9 +608,30 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // forward reports there) would leave no row for a draft, so host mode verifies the device's 16. ServeBase base(*e.fwd, int(npu::token_bound(logits.size(), e.tok->size())), e.host_gemm ? 16 : e.fwd->max_batch_rows()); - npu::mtp::PromptLookupHead head(ids); + npu::mtp::PromptLookupHead lookup(ids); + std::unique_ptr dflash; + if (e.draft) { + const int bound = int(npu::token_bound(logits.size(), e.tok->size())); + npu::dflash::HeadHooks hooks; + hooks.embed = [&e](int t, std::vector& row) { + e.fwd->embed_row(t, row); + return !row.empty(); + }; + // The target's own head on the drafted row, then the package's rule: DFlash 2's selector when it has + // one, else the head's choice, bounded like sample_token(). + hooks.pick = [&e, bound](const float* h, int prev) { + std::vector hv(h, h + e.draft_w.hidden), lg; + if (!e.fwd->lm_head_logits(hv, lg) || lg.empty()) return -1; + if (e.draft->cfg.generation >= 2) return npu::dflash::dflash2_select(*e.draft, e.draft_hproj, h, prev, lg, bound); + return npu::argmax_token(lg.data(), lg.size(), bound); + }; + dflash = std::make_unique(e.draft_w, e.draft_taps, hooks, + int(e.draft->cfg.mask_token_id), + e.draft->cfg.block_size, e.draft->cfg.sliding_window); + } + npu::mtp::DraftHead& head = dflash ? static_cast(*dflash) : lookup; npu::mtp::Options o; - o.k = e.spec_lookup; + o.k = e.draft ? e.spec_draft_k : e.spec_lookup; o.max_tokens = limit; o.tau = 0.0f; o.fallback_below = e.spec_fallback; @@ -609,8 +642,8 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib return deliver(t); }); if (!r.ok) throw std::runtime_error("speculative decode: " + r.err); - std::fprintf(stderr, "[spec] %d tokens: %d rounds, %d of %d drafts accepted, %d plain steps%s\n", - int(r.tokens.size()), r.stats.batches, r.stats.accepted, r.stats.drafted, r.stats.steps, + std::fprintf(stderr, "[spec] %s %d tokens: %d rounds, %d of %d drafts accepted, %d plain steps%s\n", + e.draft ? "dflash" : "lookup", int(r.tokens.size()), r.stats.batches, r.stats.accepted, r.stats.drafted, r.stats.steps, r.stats.fell_back_at >= 0 ? (", fell back after round " + std::to_string(r.stats.fell_back_at)).c_str() : ""); } @@ -1122,6 +1155,8 @@ int run_forward_serve(int argc, char** argv) { int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's) int spec_lookup = 0; // --spec-lookup K (engine#1781): prompt-lookup drafts, K per verify batch double spec_fallback = 0.0; // --spec-fallback X: stop drafting below X tokens per round + std::string spec_draft; // --spec-draft : a DFlash draft package as the drafter + int spec_draft_k = 0; // --spec-draft-k K: drafts per round (default: the package's block - 1) for (int i = 0; i < argc; i++) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -1143,10 +1178,15 @@ int run_forward_serve(int argc, char** argv) { else if (a == "--dump-logits") dump_logits = next(); else if (a == "--spec-lookup") spec_lookup = std::stoi(next()); else if (a == "--spec-fallback") spec_fallback = std::stod(next()); + else if (a == "--spec-draft") spec_draft = next(); + else if (a == "--spec-draft-k") spec_draft_k = std::stoi(next()); else throw std::runtime_error("unknown option " + a); } if (model.empty()) - throw std::runtime_error("usage: 1bit forward-serve -m --kernels [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits ] [--spec-lookup K [--spec-fallback X]]"); + throw std::runtime_error("usage: 1bit forward-serve -m --kernels [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits ] [--spec-lookup K | --spec-draft [--spec-draft-k K]] [--spec-fallback X]"); + if (!spec_draft.empty() && spec_lookup > 0) + throw std::runtime_error("--spec-draft and --spec-lookup are two drafters; give one"); + if (spec_draft_k != 0 && spec_draft.empty()) throw std::runtime_error("--spec-draft-k needs --spec-draft"); if (spec_lookup < 0 || spec_lookup > 15) throw std::runtime_error("--spec-lookup takes 0..15 drafts per round (a verify batch holds at most 16 rows), not " + std::to_string(spec_lookup)); @@ -1458,6 +1498,30 @@ int run_forward_serve(int argc, char** argv) { // "hung". Measured cost is 44.5 s on a folded 27B pack, and `tests/serve_port.sh`'s helper defaults to a 30 s // budget -- a caller that only knows about `ready` would time out and report "serve never reported a port", // which is the misleading shape #1531/#1545 spent effort removing. + if (!spec_draft.empty()) { + // The draft is loaded once and checked against this target before the server says ready: a package + // for another model would otherwise fail only at the first greedy request's first draft. + auto dm = std::make_unique(npu::dflash::load_draft(spec_draft)); + std::vector row; + e.fwd->embed_row(0, row); + if (int(row.size()) != dm->cfg.hidden) + throw std::runtime_error("--spec-draft " + spec_draft + ": the draft is " + std::to_string(dm->cfg.hidden) + + " wide, this model " + std::to_string(row.size())); + if (!e.fwd->set_layer_taps(dm->cfg.target_layer_ids, &e.draft_taps)) + throw std::runtime_error("--spec-draft " + spec_draft + ": its target layers do not fit this model"); + const int most = dm->cfg.block_size - 1; + if (most < 1) throw std::runtime_error("--spec-draft " + spec_draft + ": block size " + std::to_string(most + 1)); + if (spec_draft_k < 0 || spec_draft_k > most) + throw std::runtime_error("--spec-draft-k takes 1.." + std::to_string(most) + " for this package"); + e.spec_draft_k = spec_draft_k > 0 ? spec_draft_k : most; + e.draft_w = npu::dflash::decode_draft(*dm); + if (dm->cfg.generation >= 2) e.draft_hproj = npu::dflash::decode_plain(dm->selector_hproj); + std::fprintf(stderr, "1bit forward-serve: speculative drafts from %s (DFlash %d, %d layers, taps", + spec_draft.c_str(), dm->cfg.generation, dm->cfg.layers); + for (int l : dm->cfg.target_layer_ids) std::fprintf(stderr, " %d", l); + std::fprintf(stderr, "), up to %d per round\n", e.spec_draft_k); + e.draft = std::move(dm); + } std::fprintf(stderr, "1bit forward-serve: warming (weight cache; first request would otherwise pay it)\n"); std::fflush(stderr); const auto t0 = std::chrono::steady_clock::now(); diff --git a/docs/npu.md b/docs/npu.md index f520cf8a5..1c0615854 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3439,6 +3439,17 @@ text, `finish_reason` and `completion_tokens` for: In host-GEMM mode the verify batch is 16 rows, the device's width. Host GEMMs have no width of their own, and the 1 the forward reports there would leave no row for a draft. The device route has not been measured with it yet. +**`forward-serve --spec-draft `** (`--spec-draft-k K`, default the package's block minus one) serves the same path +with a DFlash draft package as the drafter, in place of the prompt lookup. At load the package is decoded once and +checked against the target, before `ready`: the widths must match, and its target layers must exist. The forward then +writes those layers' hidden states into a tap sink on every pass. The sink is emptied when a request takes the engine, +so its rows start at the request's first position. `DFlashHead` drafts from it, picking with the target's own head: +DFlash 2's candidate selector when the package has one (`dflash2_select`, `npu/dflash_head.h`), else the bounded argmax. +A request that resumed from a saved prefix has no rows for the positions it skipped, so its head refuses to draft and +it decodes with plain steps: the same text, just no speedup. The same test serves a generation-1 package it writes +(`tests/dflash_draft_test.cpp`'s synthetic layout, sized to the fabricated target). On it every greedy request of two +or more tokens drafts, and every reply equals the plain server's. + **What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth 6 measured 0 %). diff --git a/npu/dflash_head.h b/npu/dflash_head.h index f835c862d..b18354b41 100644 --- a/npu/dflash_head.h +++ b/npu/dflash_head.h @@ -50,6 +50,35 @@ struct HeadHooks { std::function pick; }; +// DFlash 2's candidate selector at one drafted row (the pick rule of a generation-2 package). `logits` is the target +// head on the row `h`; the top `selector_top_k` ids below `bound` (the tokenizer's size; 0 = all) are the candidates, +// each re-scored by selector_pick() with the pairwise term against `prev`. `hproj` is decode_plain(m.selector_hproj). +// Returns -1 when the package has no selector or there are fewer candidates than it wants. +inline int dflash2_select(const DraftModel& m, const std::vector& hproj, const float* h, int prev, + const std::vector& logits, int bound = 0) { + const int k = m.cfg.selector_top_k, R = m.cfg.selector_rank, H = m.cfg.hidden; + const int n = (bound > 0 && size_t(bound) < logits.size()) ? bound : int(logits.size()); + if (k <= 0 || R <= 0 || n < k || hproj.size() != size_t(R) * size_t(H)) return -1; + std::vector idx(static_cast(n)); + for (int j = 0; j < n; j++) idx[size_t(j)] = j; + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { + return logits[size_t(a)] > logits[size_t(b)] || (logits[size_t(a)] == logits[size_t(b)] && a < b); + }); + std::vector hp(static_cast(R)), prow(static_cast(R)), succ(static_cast(k) * static_cast(R)), unary(static_cast(k)); + for (int r = 0; r < R; r++) { + double acc = 0.0; + const float* wr = hproj.data() + size_t(r) * size_t(H); + for (int q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]); + hp[size_t(r)] = float(acc); + } + decode_row(m.selector_pred, prev, prow.data()); + for (int c = 0; c < k; c++) { + unary[size_t(c)] = logits[size_t(idx[size_t(c)])]; + decode_row(m.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * size_t(R)); + } + return idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), R))]; +} + class DFlashHead : public mtp::DraftHead { public: // `taps` is the forward's tap sink, one row of taps * hidden floats per position, starting at position 0. diff --git a/tests/serve_spec_lookup_test.py b/tests/serve_spec_lookup_test.py index 4a92fcd94..a9613ac2c 100644 --- a/tests/serve_spec_lookup_test.py +++ b/tests/serve_spec_lookup_test.py @@ -21,7 +21,8 @@ prompt-lookup drafter. It is only allowed where it cannot change a token, and this holds it to that end to end: the two-layer qwen3 container tests/batch_check_state_ci.py fabricates, a 256-token byte-level tokenizer written here, and `1bit forward-serve` in host-GEMM mode (NPU_INFER_HOST_GEMM=1: no device is opened, no lock is taken) started twice, -plain and with --spec-lookup 4. The same id prompts go to both: several budgets, stop sequences cut out of the plain +plain, with --spec-lookup 4, and with --spec-draft on a DFlash package written here (tests/dflash_draft_test.cpp's +synthetic layout, sized to the target). The same id prompts go to both: several budgets, stop sequences cut out of the plain replies (so they do end their replies), and one seeded sampled request. Text, finish_reason and completion_tokens must be equal; the spec server must take the speculative path on exactly the greedy requests and draft (its `[spec]` lines). @@ -32,6 +33,7 @@ import json import os import socket +import struct import subprocess import sys import tempfile @@ -65,6 +67,103 @@ def write_tokenizer(d): json.dump(tok, f) +# A generation-1 DFlash package for the fabricated target, after tests/dflash_draft_test.cpp's synthetic one: the same +# formulas and layout (q4_1 tile grids of 5120 B, a BF16 fusion in 32-row blocks), sized to this target (hidden 256, +# taps on its layers 0 and 1). +def to_bf16(f): + return struct.unpack("> 16 + + +def code_of(salt, n, k): + return (n * 7 + k * 3 + salt) & 15 + + +def scale_of(salt, n, g): + return 0.0078125 * (1 + ((n + g + salt) % 4)) + + +def min_of(salt, n, g): + return -0.0625 * (1 + ((n * 3 + g + salt) % 5)) + + +def encode_projection(salt, N, K): + out = bytearray() + for tr in range(N // 32): + for tc in range(K // 256): + tile = bytearray(5120) + for g in range(8): + for r in range(32): + n, grp = tr * 32 + r, tc * 8 + g + struct.pack_into("