diff --git a/CMakeLists.txt b/CMakeLists.txt index 100c69d37..652808e79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -594,6 +594,11 @@ if(TARGET OpenMP::OpenMP_CXX) target_link_libraries(mtp_draft_test PRIVATE OpenMP::OpenMP_CXX) endif() add_test(NAME mtp_draft COMMAND mtp_draft_test) +# engine#1781: the prompt-lookup drafter (npu/mtp/prompt_lookup.h), by hand and through mtp::run() on a toy base. +add_executable(prompt_lookup_test tests/prompt_lookup_test.cpp) +target_include_directories(prompt_lookup_test PRIVATE npu) +add_dependencies(onebit prompt_lookup_test) +add_test(NAME prompt_lookup COMMAND prompt_lookup_test) # engine#1781 gate 2: the distribution-following acceptance rule (accept with min(1, p/q), resample the residual at the # first rejection), with the temperature-0 identity and a deliberately wrong drafter whose output must still follow p. # Header-only and device-free; pulled into CI through npu_sampling_test for the same reason as mtp_draft_test below. @@ -2024,6 +2029,8 @@ endif() # same sources against the runtime stub (engine#1946) cannot drift from this one. onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) target_sources(onebit PRIVATE app/forward_serve.cpp) + # engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter. + target_sources(onebit PRIVATE npu/dflash_forward.cpp npu/dflash_draft.cpp) # engine#2111: the usage object's pure function, its own TU so tests/serve_usage_test.cpp can link it without # dragging in npu/rt.h (see cmake/tests-serve-usage.cmake). target_sources(onebit PRIVATE app/serve_usage.cpp) @@ -2111,6 +2118,11 @@ endif() set_tests_properties(gdn_batch_order PROPERTIES SKIP_RETURN_CODE 77) set_tests_properties(layer_taps_model PROPERTIES SKIP_RETURN_CODE 77) set_tests_properties(commit_full_accept PROPERTIES SKIP_RETURN_CODE 77) + # engine#1781: forward-serve --spec-lookup gives the plain loop's replies, end to end, host-GEMM on a container the + # script fabricates (no device, no model on disk). forward-serve exists only in this build, so it is a dev-box row. + add_test(NAME serve_spec_lookup + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/serve_spec_lookup_test.py $) + set_tests_properties(serve_spec_lookup PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600) set_tests_properties(batch_check_state PROPERTIES SKIP_RETURN_CODE 77) # The fast-lane end-to-end test (npu_lane_e2e) is retired: its kernels were FastFlowLM-derived # and its reference logits are lost. The NPU check is the private dx add-on's from-source diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 209f003ba..5fe87824d 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -54,6 +54,8 @@ #include "npu_systemone.h" #include "tokenizer.h" #include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653) +#include "../npu/mtp/prompt_lookup.h" // --spec-lookup: the speculative loop with a prompt-lookup drafter (#1781) +#include "../npu/dflash_head.h" // --spec-draft: the same loop with a DFlash draft package (#1781) #include "rt.h" // the own NPU runtime (ONEBIT_NPU_RT_DIR) #include @@ -139,6 +141,21 @@ struct ForwardEngine { // KL-divergence comparison against another stack (engine#671 gate b) needs the distribution // itself. Empty means off, which is the default: nothing is written unless asked. std::string dump_logits; + // --spec-lookup K (engine#1781): a greedy request without grammar or logprobs decodes through mtp::run_from() with + // the prompt-lookup drafter, up to K drafts verified per batch; 0 = off (the default). --spec-fallback X: stop + // drafting for the rest of a reply once its last 3 rounds averaged fewer than X tokens (0 = never). + int spec_lookup = 0; + double spec_fallback = 0.0; + bool host_gemm = false; // NPU_INFER_HOST_GEMM: the forward runs on the host, with no launch width of its own + // --spec-draft (engine#1781): a DFlash draft package as the drafter instead of the prompt lookup, up to + // spec_draft_k drafts a round. The forward writes the draft's tap layers into draft_taps on every pass; the sink + // is emptied at the start of each request, so a request that resumed from a saved prefix has no rows for the + // positions it did not run, and its head refuses to draft (it decodes with plain steps, the same text). + std::unique_ptr draft; + npu::dflash::DraftWeights draft_w; + std::vector draft_hproj; // generation 2: the selector's hidden projection, decoded once + std::vector draft_taps; + int spec_draft_k = 0; // The dump's response index. `step_.bin` used the decode index alone, so a second request overwrote the first // request's steps: a multi-prompt comparison (the KLD row, engine#803) then silently compared only the LAST prompt's // dumps and looked complete. Numbering the response too makes the set of files say how many responses it holds. @@ -346,6 +363,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (room < 1) throw ContextOverflow(ids.size(), e.max_context); const int limit = std::clamp(max_tokens, 0, room); std::lock_guard lock(e.mu); + e.draft_taps.clear(); // --spec-draft: this request's tap rows start at its first position run const auto t_request = std::chrono::steady_clock::now(); std::vector logits; @@ -529,28 +547,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib }; bool stopped_at_eos = false, stopped_at_stop = false; int produced = 0; - for (int n = 0; n < limit; n++) { - // the client is gone: free the NPU (the bridge knows first when a write to it failed) - if (req.is_connection_closed() || (bridge && bridge->cancelled())) break; - if (gram) refresh_allowed(); - // Raw distribution for a reference comparison: sample_token() takes logits by const - // reference and the copy below happens before it runs, so what lands on disk is what - // the model produced, not what the sampler chose. - if (!e.dump_logits.empty()) { - const std::string path = e.dump_logits + "/step_" + std::to_string(dump_req) + "_" + - std::to_string(n) + ".bin"; - if (std::FILE* f = onebit_fopen_private_wb(path.c_str())) { // 0600, no symlink (file_create.h) - std::fwrite(logits.data(), sizeof(float), logits.size(), f); - std::fclose(f); - } - } - const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size()); - // -1: the grammar allows no token here, so the answer ends where it is. An end token is - // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply - // that completes reports "stop" instead of padding to the budget (engine#1182). - if (next < 0) { stopped_at_eos = true; break; } + // One chosen token through the reply: the end tokens, the grammar, the stop sequences and the hold-back that + // keeps a partial stop match unsent. false = the reply ends here (an end token, a stop sequence, or a client + // that stopped reading). The plain loop and the speculative one hand every token to this, in order. + auto deliver = [&](int next) -> bool { if (std::find(e.eos.begin(), e.eos.end(), next) != e.eos.end() || - (e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; break; } + (e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; return false; } if (gram) gram->feed(grammar_state, e.tok_text[next]); const std::string piece = e.tok->decode(next); pending += piece; @@ -564,7 +566,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (stop_at != std::string::npos) { if (stop_at > 0) on_text(pending.substr(0, stop_at), take_held(stop_at)); stopped_at_stop = true; - break; + return false; } // Send what no stop sequence can still match, and keep the tail: the next // token may complete a match that starts in it. @@ -575,8 +577,97 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib if (emit > 0) { const std::string out = pending.substr(0, emit); pending.erase(0, emit); - if (!on_text(out, take_held(emit))) break; + if (!on_text(out, take_held(emit))) return false; + } + return true; + }; + // engine#1781: the speculative path, only where it cannot change a token. Greedy, so each token is the bounded + // argmax the verify rows check drafts against; no grammar (its mask is per token and per state); no logprobs + // and no logit dump (both read the logits of every position, and a verify batch's rows are not kept). The text + // is then the plain loop's token for token -- what tests/serve_spec_lookup_test.py holds it to. + const bool spec = + (e.spec_lookup > 0 || e.draft) && opts.sample.greedy() && !gram && !want_lp && e.dump_logits.empty(); + if (spec) { + struct ServeBase : npu::mtp::BaseModel { + Q4nxNpuForward& f; + int nv, rows; + ServeBase(Q4nxNpuForward& fw, int v, int r) : f(fw), nv(v), rows(r) {} + int vocab() const override { return nv; } + int hidden_size() const override { return int(f.last_hidden().size()); } + int max_batch_rows() const override { return rows; } + bool step(int t, int p, std::vector& l) override { return f.step(t, p, l); } + bool step_batch_all(const std::vector& t, int s0, std::vector& l) override { + f.begin_verify(int(t.size()), s0); + return f.step_batch_all(t, s0, l); + } + bool commit_verify(int a) override { return f.commit_verify_rows(a); } + const std::vector& last_hidden() const override { return f.last_hidden(); } + }; + // The bound sample_token() applies: ids past the tokenizer are padding and never chosen (engine#2017). + // The device verifies as many rows as its kernels are wide; host GEMMs have no width, and 1 (what the + // forward reports there) would leave no row for a draft, so host mode verifies the device's 16. + ServeBase base(*e.fwd, int(npu::token_bound(logits.size(), e.tok->size())), + e.host_gemm ? 16 : e.fwd->max_batch_rows()); + npu::mtp::PromptLookupHead lookup(ids); + std::unique_ptr dflash; + if (e.draft) { + const int bound = int(npu::token_bound(logits.size(), e.tok->size())); + npu::dflash::HeadHooks hooks; + hooks.embed = [&e](int t, std::vector& row) { + e.fwd->embed_row(t, row); + return !row.empty(); + }; + // The target's own head on the drafted row, then the package's rule: DFlash 2's selector when it has + // one, else the head's choice, bounded like sample_token(). + hooks.pick = [&e, bound](const float* h, int prev) { + std::vector hv(h, h + e.draft_w.hidden), lg; + if (!e.fwd->lm_head_logits(hv, lg) || lg.empty()) return -1; + if (e.draft->cfg.generation >= 2) return npu::dflash::dflash2_select(*e.draft, e.draft_hproj, h, prev, lg, bound); + return npu::argmax_token(lg.data(), lg.size(), bound); + }; + dflash = std::make_unique(e.draft_w, e.draft_taps, hooks, + int(e.draft->cfg.mask_token_id), + e.draft->cfg.block_size, e.draft->cfg.sliding_window); } + npu::mtp::DraftHead& head = dflash ? static_cast(*dflash) : lookup; + npu::mtp::Options o; + o.k = e.draft ? e.spec_draft_k : e.spec_lookup; + o.max_tokens = limit; + o.tau = 0.0f; + o.fallback_below = e.spec_fallback; + o.consume_last = false; // the plain loop does not run the reply's last token either + const npu::mtp::Result r = npu::mtp::run_from( + base, &head, int(ids.size()), logits, o, [&](int t) { + if (req.is_connection_closed() || (bridge && bridge->cancelled())) return false; + return deliver(t); + }); + if (!r.ok) throw std::runtime_error("speculative decode: " + r.err); + std::fprintf(stderr, "[spec] %s %d tokens: %d rounds, %d of %d drafts accepted, %d plain steps%s\n", + e.draft ? "dflash" : "lookup", int(r.tokens.size()), r.stats.batches, r.stats.accepted, r.stats.drafted, r.stats.steps, + r.stats.fell_back_at >= 0 ? (", fell back after round " + std::to_string(r.stats.fell_back_at)).c_str() + : ""); + } + for (int n = 0; !spec && n < limit; n++) { + // the client is gone: free the NPU (the bridge knows first when a write to it failed) + if (req.is_connection_closed() || (bridge && bridge->cancelled())) break; + if (gram) refresh_allowed(); + // Raw distribution for a reference comparison: sample_token() takes logits by const + // reference and the copy below happens before it runs, so what lands on disk is what + // the model produced, not what the sampler chose. + if (!e.dump_logits.empty()) { + const std::string path = e.dump_logits + "/step_" + std::to_string(dump_req) + "_" + + std::to_string(n) + ".bin"; + if (std::FILE* f = onebit_fopen_private_wb(path.c_str())) { // 0600, no symlink (file_create.h) + std::fwrite(logits.data(), sizeof(float), logits.size(), f); + std::fclose(f); + } + } + const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size()); + // -1: the grammar allows no token here, so the answer ends where it is. An end token is + // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply + // that completes reports "stop" instead of padding to the budget (engine#1182). + if (next < 0) { stopped_at_eos = true; break; } + if (!deliver(next)) break; // No further token will be sampled after this one (limit reached), so the // decode step that would only produce the NEXT position's logits is a whole // weight stream -- 8.7 s on the 27B -- spent on a result nobody reads. At @@ -1062,6 +1153,10 @@ int run_forward_serve(int argc, char** argv) { npu::PrefixOptions prefix_opts; // --prefix-slots / --prefix-cache-mb / --prefix-min-gain (#650) bool prefix_pin = false; // --prefix-pin (engine#1785) int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's) + int spec_lookup = 0; // --spec-lookup K (engine#1781): prompt-lookup drafts, K per verify batch + double spec_fallback = 0.0; // --spec-fallback X: stop drafting below X tokens per round + std::string spec_draft; // --spec-draft : a DFlash draft package as the drafter + int spec_draft_k = 0; // --spec-draft-k K: drafts per round (default: the package's block - 1) for (int i = 0; i < argc; i++) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -1081,10 +1176,21 @@ int run_forward_serve(int argc, char** argv) { else if (a == "--npu-streams") streams = std::stoi(next()); else if (a == "--npu-target") npu_target = next(); else if (a == "--dump-logits") dump_logits = next(); + else if (a == "--spec-lookup") spec_lookup = std::stoi(next()); + else if (a == "--spec-fallback") spec_fallback = std::stod(next()); + else if (a == "--spec-draft") spec_draft = next(); + else if (a == "--spec-draft-k") spec_draft_k = std::stoi(next()); else throw std::runtime_error("unknown option " + a); } if (model.empty()) - throw std::runtime_error("usage: 1bit forward-serve -m --kernels [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits ]"); + throw std::runtime_error("usage: 1bit forward-serve -m --kernels [-p PORT] [--npu-streams N] [--npu-target xdna] [--dump-logits ] [--spec-lookup K | --spec-draft [--spec-draft-k K]] [--spec-fallback X]"); + if (!spec_draft.empty() && spec_lookup > 0) + throw std::runtime_error("--spec-draft and --spec-lookup are two drafters; give one"); + if (spec_draft_k != 0 && spec_draft.empty()) throw std::runtime_error("--spec-draft-k needs --spec-draft"); + if (spec_lookup < 0 || spec_lookup > 15) + throw std::runtime_error("--spec-lookup takes 0..15 drafts per round (a verify batch holds at most 16 rows), not " + + std::to_string(spec_lookup)); + if (!(spec_fallback >= 0.0)) throw std::runtime_error("--spec-fallback takes a number of tokens >= 0"); if (streams < 1 || streams > kNpuMaxStreams) throw std::runtime_error("--npu-streams takes 1.." + std::to_string(kNpuMaxStreams) + " (the device's hwctx limit), not " + std::to_string(streams)); @@ -1104,6 +1210,8 @@ int run_forward_serve(int argc, char** argv) { ForwardEngine e; e.id = alias.empty() ? std::filesystem::path(dir).filename().string() : alias; + e.spec_lookup = spec_lookup; + e.spec_fallback = spec_fallback; if (!dump_logits.empty()) { std::error_code ec; std::filesystem::create_directories(dump_logits, ec); @@ -1261,6 +1369,7 @@ int run_forward_serve(int argc, char** argv) { // other lanes' device work. Open the device, and the lock, only when the forward is // going to use it. const bool host_gemm = std::getenv("NPU_INFER_HOST_GEMM") != nullptr; + e.host_gemm = host_gemm; if (npu_lock_taken(!host_gemm, std::getenv("NPU_NO_DEVICE_LOCK") != nullptr)) e.lock_fd = take_npu_device_lock(streams); std::unique_ptr dev; @@ -1389,6 +1498,30 @@ int run_forward_serve(int argc, char** argv) { // "hung". Measured cost is 44.5 s on a folded 27B pack, and `tests/serve_port.sh`'s helper defaults to a 30 s // budget -- a caller that only knows about `ready` would time out and report "serve never reported a port", // which is the misleading shape #1531/#1545 spent effort removing. + if (!spec_draft.empty()) { + // The draft is loaded once and checked against this target before the server says ready: a package + // for another model would otherwise fail only at the first greedy request's first draft. + auto dm = std::make_unique(npu::dflash::load_draft(spec_draft)); + std::vector row; + e.fwd->embed_row(0, row); + if (int(row.size()) != dm->cfg.hidden) + throw std::runtime_error("--spec-draft " + spec_draft + ": the draft is " + std::to_string(dm->cfg.hidden) + + " wide, this model " + std::to_string(row.size())); + if (!e.fwd->set_layer_taps(dm->cfg.target_layer_ids, &e.draft_taps)) + throw std::runtime_error("--spec-draft " + spec_draft + ": its target layers do not fit this model"); + const int most = dm->cfg.block_size - 1; + if (most < 1) throw std::runtime_error("--spec-draft " + spec_draft + ": block size " + std::to_string(most + 1)); + if (spec_draft_k < 0 || spec_draft_k > most) + throw std::runtime_error("--spec-draft-k takes 1.." + std::to_string(most) + " for this package"); + e.spec_draft_k = spec_draft_k > 0 ? spec_draft_k : most; + e.draft_w = npu::dflash::decode_draft(*dm); + if (dm->cfg.generation >= 2) e.draft_hproj = npu::dflash::decode_plain(dm->selector_hproj); + std::fprintf(stderr, "1bit forward-serve: speculative drafts from %s (DFlash %d, %d layers, taps", + spec_draft.c_str(), dm->cfg.generation, dm->cfg.layers); + for (int l : dm->cfg.target_layer_ids) std::fprintf(stderr, " %d", l); + std::fprintf(stderr, "), up to %d per round\n", e.spec_draft_k); + e.draft = std::move(dm); + } std::fprintf(stderr, "1bit forward-serve: warming (weight cache; first request would otherwise pay it)\n"); std::fflush(stderr); const auto t0 = std::chrono::steady_clock::now(); diff --git a/docs/npu.md b/docs/npu.md index 99475c43e..1c0615854 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -3421,6 +3421,35 @@ fewer than `fallback_below` tokens on average, the run finishes with plain steps round). `tools/dflash2_probe loop` with `DFLASH2_MTP_RUN=1` drives the served 27B through exactly these pieces, with `DFlashHead` and the DFlash 2 selector as its pick hook, after the same batched prefill as its hand-written loop. +**Serving it: `forward-serve --spec-lookup K`** (off by default; `--spec-fallback X` adds the fallback). A request that +is greedy, has no grammar, and asks for no logprobs (and a server without `--dump-logits`) decodes through +`mtp::run_from()` with the prompt-lookup drafter (`npu/mtp/prompt_lookup.h`). That drafter needs no model: it proposes +the up to `K` tokens that followed the latest earlier occurrence of the stream's last one to three tokens. Every +other request takes the plain loop. The rule is that speculation is used only where it cannot change a token: each +token is still the tokenizer-bounded argmax, and both loops hand every token to the same code for end tokens, stop +sequences and the stop hold-back. The reply's last token is not run through the model (`Options::consume_last = +false`), as in the plain loop. The server logs one `[spec]` line per request (rounds, drafts accepted, plain steps). +`tests/serve_spec_lookup_test.py` (ctest `serve_spec_lookup`, on a build with `ONEBIT_NPU`) holds it to that end to +end. It fabricates a container, serves it host-GEMM twice (plain, then with `--spec-lookup 4`), and requires the same +text, `finish_reason` and `completion_tokens` for: +- every budget; +- stop sequences cut from the plain replies, so they do end them; +- a seeded sampled request, which must stay on the plain loop. + +In host-GEMM mode the verify batch is 16 rows, the device's width. Host GEMMs have no width of their own, and the 1 +the forward reports there would leave no row for a draft. The device route has not been measured with it yet. + +**`forward-serve --spec-draft `** (`--spec-draft-k K`, default the package's block minus one) serves the same path +with a DFlash draft package as the drafter, in place of the prompt lookup. At load the package is decoded once and +checked against the target, before `ready`: the widths must match, and its target layers must exist. The forward then +writes those layers' hidden states into a tap sink on every pass. The sink is emptied when a request takes the engine, +so its rows start at the request's first position. `DFlashHead` drafts from it, picking with the target's own head: +DFlash 2's candidate selector when the package has one (`dflash2_select`, `npu/dflash_head.h`), else the bounded argmax. +A request that resumed from a saved prefix has no rows for the positions it skipped, so its head refuses to draft and +it decodes with plain steps: the same text, just no speedup. The same test serves a generation-1 package it writes +(`tests/dflash_draft_test.cpp`'s synthetic layout, sized to the fabricated target). On it every greedy request of two +or more tokens drafts, and every reply equals the plain server's. + **What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth 6 measured 0 %). diff --git a/npu/dflash_head.h b/npu/dflash_head.h index f835c862d..b18354b41 100644 --- a/npu/dflash_head.h +++ b/npu/dflash_head.h @@ -50,6 +50,35 @@ struct HeadHooks { std::function pick; }; +// DFlash 2's candidate selector at one drafted row (the pick rule of a generation-2 package). `logits` is the target +// head on the row `h`; the top `selector_top_k` ids below `bound` (the tokenizer's size; 0 = all) are the candidates, +// each re-scored by selector_pick() with the pairwise term against `prev`. `hproj` is decode_plain(m.selector_hproj). +// Returns -1 when the package has no selector or there are fewer candidates than it wants. +inline int dflash2_select(const DraftModel& m, const std::vector& hproj, const float* h, int prev, + const std::vector& logits, int bound = 0) { + const int k = m.cfg.selector_top_k, R = m.cfg.selector_rank, H = m.cfg.hidden; + const int n = (bound > 0 && size_t(bound) < logits.size()) ? bound : int(logits.size()); + if (k <= 0 || R <= 0 || n < k || hproj.size() != size_t(R) * size_t(H)) return -1; + std::vector idx(static_cast(n)); + for (int j = 0; j < n; j++) idx[size_t(j)] = j; + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { + return logits[size_t(a)] > logits[size_t(b)] || (logits[size_t(a)] == logits[size_t(b)] && a < b); + }); + std::vector hp(static_cast(R)), prow(static_cast(R)), succ(static_cast(k) * static_cast(R)), unary(static_cast(k)); + for (int r = 0; r < R; r++) { + double acc = 0.0; + const float* wr = hproj.data() + size_t(r) * size_t(H); + for (int q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]); + hp[size_t(r)] = float(acc); + } + decode_row(m.selector_pred, prev, prow.data()); + for (int c = 0; c < k; c++) { + unary[size_t(c)] = logits[size_t(idx[size_t(c)])]; + decode_row(m.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * size_t(R)); + } + return idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), R))]; +} + class DFlashHead : public mtp::DraftHead { public: // `taps` is the forward's tap sink, one row of taps * hidden floats per position, starting at position 0. diff --git a/npu/mtp/draft.h b/npu/mtp/draft.h index df2fd4f89..3f1546043 100644 --- a/npu/mtp/draft.h +++ b/npu/mtp/draft.h @@ -173,6 +173,10 @@ struct Options { // fallback_below <= 0 turns it off. double fallback_below = 0.0; int fallback_window = 3; + // When the budget ends on the base's own token, whether that token is still run through the base. true keeps the + // base holding every emitted token (what the tests and a caller that continues the stream want); false skips a + // pass whose logits nobody reads, which is what forward-serve does at the end of a reply. + bool consume_last = true; }; struct Stats { @@ -277,6 +281,7 @@ inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector= opt.max_tokens) break; if (k <= 0 || fallback.off()) { // plain decode (no drafter, k = 0, or the fallback stopped it) if (!base.step(t1, pos++, logits)) { r.err = "step failed"; return r; } @@ -346,16 +351,16 @@ inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector& rows = base.last_hidden_rows(); const size_t h = (size_t)base.hidden_size(); const size_t got = (h && rows.size() % h == 0) ? rows.size() / h : 0; - if (!rows.empty()) { - if (got != (size_t)(take + 1) && got != (size_t)(nd + 1)) { - r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " + - std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)"; - return r; - } - std::vector kept{t1}; - kept.insert(kept.end(), d.begin(), d.begin() + take); - head->commit_rows(kept, rows, take + 1); + if (!rows.empty() && got != (size_t)(take + 1) && got != (size_t)(nd + 1)) { + r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " + + std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)"; + return r; } + // Always, even when the base keeps no rows (`rows` is then empty): a drafter that reads only the tokens + // (the prompt-lookup head) follows the stream through these calls alone. + std::vector kept{t1}; + kept.insert(kept.end(), d.begin(), d.begin() + take); + head->commit_rows(kept, rows, take + 1); r.stats.ms_commit += lap(tp); logits.assign(all.begin() + (size_t)take * (size_t)V, all.begin() + (size_t)(take + 1) * (size_t)V); pos += take + 1; diff --git a/npu/mtp/prompt_lookup.h b/npu/mtp/prompt_lookup.h new file mode 100644 index 000000000..2d07a7d18 --- /dev/null +++ b/npu/mtp/prompt_lookup.h @@ -0,0 +1,78 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// prompt_lookup.h -- a drafter with no model: propose what followed the latest earlier occurrence of the stream's +// last tokens (engine#1781). +// +// The stream is the prompt plus every committed token, the last emitted token (t1) included. To draft, take its last n +// tokens (n from max_ngram down to min_ngram, the first length that matches wins), find their latest earlier +// occurrence, and propose the tokens that followed it. A reply that repeats its prompt -- code being edited, a quoted +// passage, a list that follows a pattern -- is then drafted at no cost but the verify batch, and a reply that does not +// gets no proposal at all, so the round falls back to a plain step. It never reads the base's hidden, so mtp::run() +// takes the one-pass round (needs_last_hidden() is false), and mtp::run's fallback bounds what a run of misses costs. +#pragma once + +#include +#include + +#include "draft.h" + +namespace onebit::npu::mtp { + +class PromptLookupHead : public DraftHead { +public: + explicit PromptLookupHead(std::vector prompt, int max_ngram = 3, int min_ngram = 1) + : prompt_(std::move(prompt)), stream_(prompt_), max_n_(max_ngram), min_n_(min_ngram < 1 ? 1 : min_ngram) {} + + bool needs_last_hidden() const override { return false; } + + int draft(const std::vector& /*hidden*/, int last_token, int pos, int k, std::vector& out, + std::vector* conf = nullptr) override { + out.clear(); + if (conf) conf->clear(); // no probabilities: mtp::run skips its tau gate + // last_token runs at pos - 1, right after the committed stream; anything else means the stream is not the + // base's, and a proposal from it would be a guess about some other text. + if (k <= 0 || pos != int(stream_.size()) + 1) return 0; + stream_.push_back(last_token); + const int len = int(stream_.size()); + for (int n = std::min(max_n_, len - 1); n >= min_n_; n--) { + const int* suf = stream_.data() + (len - n); + for (int i = len - n - 1; i >= 0; i--) { // latest earlier occurrence first + bool hit = true; + for (int j = 0; j < n && hit; j++) hit = stream_[size_t(i + j)] == suf[j]; + if (!hit) continue; + for (int j = i + n; j < len && int(out.size()) < k; j++) out.push_back(stream_[size_t(j)]); + break; + } + if (!out.empty()) break; + } + stream_.pop_back(); + return int(out.size()); + } + + void commit_rows(const std::vector& tokens, const std::vector& /*rows*/, int M) override { + for (int i = 0; i < M && i < int(tokens.size()); i++) stream_.push_back(tokens[size_t(i)]); + } + + void reset() override { stream_ = prompt_; } + + const std::vector& stream() const { return stream_; } + +private: + std::vector prompt_, stream_; + int max_n_, min_n_; +}; + +} // namespace onebit::npu::mtp diff --git a/tests/mtp_draft_test.cpp b/tests/mtp_draft_test.cpp index dc6dd6a6f..e98685d8b 100644 --- a/tests/mtp_draft_test.cpp +++ b/tests/mtp_draft_test.cpp @@ -1087,6 +1087,25 @@ int main() { const Result re = onebit::npu::mtp::run(eq, &oh, P, f); check(re.ok && same(re.tokens, ref.tokens) && re.stats.fell_back_at == -1, "fallback: rounds of exactly 2 tokens against a bar of 2.0 keep drafting"); + // consume_last = false: a run whose budget ends on the base's own token does not pay the pass after it. + for (int kind : {0, 1}) { + ToyBase m1(8), m2(8); + OnePassHead h1((OnePassHead::Kind)kind, (int)P.size()), h2((OnePassHead::Kind)kind, (int)P.size()); + Options c = base_opt; + c.k = 4; + const Result a1 = onebit::npu::mtp::run(m1, &h1, P, c); + c.consume_last = false; + const Result a2 = onebit::npu::mtp::run(m2, &h2, P, c); + const int passes1 = m1.steps() + m1.batches(), passes2 = m2.steps() + m2.batches(); + char msg[200]; + std::snprintf(msg, sizeof msg, "consume_last=false, %s head: same text, %d passes against %d, base at %d of %d", + kind == 0 ? "perfect" : "garbage", passes2, passes1, m2.consumed(), m1.consumed()); + // perfect at k=4 ends inside a batch (nothing to skip); garbage ends on the base's own token (one skipped) + const int skipped = kind == 0 ? 0 : 1; + check(a1.ok && a2.ok && same(a1.tokens, a2.tokens) && passes2 == passes1 - skipped && + m2.consumed() == m1.consumed() - skipped, + msg); + } } std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails); diff --git a/tests/prompt_lookup_test.cpp b/tests/prompt_lookup_test.cpp new file mode 100644 index 000000000..962b5ceb6 --- /dev/null +++ b/tests/prompt_lookup_test.cpp @@ -0,0 +1,147 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// prompt_lookup_test.cpp -- the prompt-lookup drafter (npu/mtp/prompt_lookup.h, engine#1781). Host only, no model. +// [1] the lookup itself, by hand: the longest suffix that matches wins, its latest earlier occurrence wins, the +// continuation runs up to k tokens and may run into the suffix, nothing matches -> no proposal, a position the +// stream does not end at -> no proposal; +// [2] through mtp::run() on a toy base: the text is plain decode's whatever the base does; on a base whose answer +// repeats, drafts are accepted and the run takes fewer passes than plain decode; the head's stream is the +// committed text. + +#include "mtp/prompt_lookup.h" + +#include +#include +#include + +using onebit::npu::mtp::BaseModel; +using onebit::npu::mtp::Options; +using onebit::npu::mtp::PromptLookupHead; +using onebit::npu::mtp::Result; + +namespace { + +int fails = 0; +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", what.c_str()); + if (!ok) fails++; +} + +std::vector draft(PromptLookupHead& h, int last, int k) { + std::vector out; + h.draft({}, last, int(h.stream().size()) + 1, k, out); + return out; +} + +constexpr int V = 64; + +// next = rule(token, pos): `period` > 0 makes the answer a loop of that period, 0 makes every position different. +class Base : public BaseModel { +public: + explicit Base(int period) : period_(period) {} + int vocab() const override { return V; } + int hidden_size() const override { return 1; } + int max_batch_rows() const override { return 8; } + int next(int token, int pos) const { return period_ > 0 ? (token + 1) % period_ + 10 : (pos * 7 + 3) % V; } + bool step(int t, int p, std::vector& l) override { + if (p != consumed_) return false; + consumed_++; + steps_++; + l.assign(V, 0.0f); + l[size_t(next(t, p))] = 1.0f; + return true; + } + bool step_batch_all(const std::vector& t, int s, std::vector& all) override { + if (s != consumed_ || int(t.size()) > 8) return false; + all.assign(t.size() * size_t(V), 0.0f); + for (size_t m = 0; m < t.size(); m++) all[m * size_t(V) + size_t(next(t[m], s + int(m)))] = 1.0f; + pending_ = int(t.size()); + consumed_ += pending_; + batches_++; + return true; + } + bool commit_verify(int a) override { + if (a < 0 || a > pending_) return false; + consumed_ -= pending_ - a; + pending_ = 0; + return true; + } + const std::vector& last_hidden() const override { return h_; } + int consumed() const { return consumed_; } + int passes() const { return steps_ + batches_; } + +private: + int period_, consumed_ = 0, pending_ = 0, steps_ = 0, batches_ = 0; + std::vector h_{0.0f}; +}; + +} // namespace + +int main() { + { + PromptLookupHead h({1, 2, 3, 4, 5, 1, 2}); + check(draft(h, 3, 4) == std::vector({4, 5, 1, 2}), "suffix 1 2 3 found at the start: 4 5 1 2 follow"); + check(draft(h, 3, 9) == std::vector({4, 5, 1, 2, 3}), "k past the stream: the continuation stops at its end"); + } + { + PromptLookupHead h({7, 8, 1, 7, 8, 2, 7}); + check(draft(h, 8, 3) == std::vector({2, 7, 8}), "two earlier 7 8: the latest one's continuation (2 7 8)"); + } + { + PromptLookupHead h({3, 9, 6, 1, 9, 7, 3}); + check(draft(h, 9, 3) == std::vector({6, 1, 9}), + "3 9 matches at 0 (6 1 9), 9 alone last at 4 (7 3 9): the longer suffix wins"); + } + { + PromptLookupHead h({1, 2, 3}); + check(draft(h, 4, 4).empty(), "a token never seen before: no proposal"); + std::vector out; + check(h.draft({}, 3, 7, 4, out) == 0 && out.empty(), "a position the stream does not end at: no proposal"); + check(h.stream() == std::vector({1, 2, 3}), "drafting leaves the stream as it was"); + h.commit_rows({4, 5}, {}, 2); + check(h.stream() == std::vector({1, 2, 3, 4, 5}), "commit_rows appends the committed tokens"); + h.reset(); + check(h.stream() == std::vector({1, 2, 3}), "reset goes back to the prompt"); + } + const std::vector prompt = {10, 11, 12, 13, 14, 15, 10, 11}; + for (int period : {6, 0}) { + for (int k : {1, 3, 7}) { + Options o; + o.max_tokens = 40; + o.k = k; + o.tau = 0.0f; + Base pb(period), b(period); + const Result plain = onebit::npu::mtp::run_plain(pb, prompt, o); + PromptLookupHead h(prompt); + const Result r = onebit::npu::mtp::run(b, &h, prompt, o); + std::vector stream = prompt; + stream.insert(stream.end(), r.tokens.begin(), r.tokens.end()); + const std::string tag = "period " + std::to_string(period) + " k=" + std::to_string(k) + ": "; + check(r.ok && r.tokens == plain.tokens && b.consumed() == int(prompt.size()) + 40, + tag + "text and positions are plain decode's"); + check(h.stream() == stream, tag + "the head's stream is the committed text"); + if (period > 0) + check(r.stats.accepted > 0 && b.passes() < pb.passes(), + tag + "a repeating answer: " + std::to_string(r.stats.accepted) + " drafts accepted, " + + std::to_string(b.passes()) + " passes against plain's " + std::to_string(pb.passes())); + else + std::printf("INFO %saccepted %d of %d drafts, %d passes against plain's %d\n", tag.c_str(), + r.stats.accepted, r.stats.drafted, b.passes(), pb.passes()); + } + } + std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails); + return fails ? 1 : 0; +} diff --git a/tests/serve_spec_lookup_test.py b/tests/serve_spec_lookup_test.py new file mode 100644 index 000000000..a9613ac2c --- /dev/null +++ b/tests/serve_spec_lookup_test.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""forward-serve --spec-lookup returns the plain loop's reply, token for token (engine#1781). + +usage: tests/serve_spec_lookup_test.py <1bit binary> (needs numpy) + +The speculative path (app/forward_serve.cpp, `spec`) decodes a greedy request through mtp::run_from() with the +prompt-lookup drafter. It is only allowed where it cannot change a token, and this holds it to that end to end: the +two-layer qwen3 container tests/batch_check_state_ci.py fabricates, a 256-token byte-level tokenizer written here, and +`1bit forward-serve` in host-GEMM mode (NPU_INFER_HOST_GEMM=1: no device is opened, no lock is taken) started twice, +plain, with --spec-lookup 4, and with --spec-draft on a DFlash package written here (tests/dflash_draft_test.cpp's +synthetic layout, sized to the target). The same id prompts go to both: several budgets, stop sequences cut out of the plain +replies (so they do end their replies), and one seeded sampled request. Text, finish_reason and completion_tokens +must be equal; the spec server must take the speculative path on exactly the greedy requests and draft (its `[spec]` +lines). + +A dev-box row: forward-serve is built only with ONEBIT_NPU. Exits 77 (skip) without numpy. +""" +import importlib.util +import json +import os +import socket +import struct +import subprocess +import sys +import tempfile +import time +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.path.join(HERE, "..", "tools", "safetensors_to_q4nx.py") + + +def bytes_to_unicode(): + bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + cs = bs[:] + n = 0 + for b in range(256): + if b not in bs: + bs.append(b) + cs.append(256 + n) + n += 1 + return {b: chr(c) for b, c in zip(bs, cs)} + + +def write_tokenizer(d): + m = bytes_to_unicode() + vocab = {m[b]: b for b in range(256)} + tok = {"version": "1.0", "added_tokens": [], + "pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": False, "trim_offsets": True, "use_regex": True}, + "decoder": {"type": "ByteLevel"}, + "model": {"type": "BPE", "vocab": vocab, "merges": []}} + with open(os.path.join(d, "tokenizer.json"), "w") as f: + json.dump(tok, f) + + +# A generation-1 DFlash package for the fabricated target, after tests/dflash_draft_test.cpp's synthetic one: the same +# formulas and layout (q4_1 tile grids of 5120 B, a BF16 fusion in 32-row blocks), sized to this target (hidden 256, +# taps on its layers 0 and 1). +def to_bf16(f): + return struct.unpack("> 16 + + +def code_of(salt, n, k): + return (n * 7 + k * 3 + salt) & 15 + + +def scale_of(salt, n, g): + return 0.0078125 * (1 + ((n + g + salt) % 4)) + + +def min_of(salt, n, g): + return -0.0625 * (1 + ((n * 3 + g + salt) % 5)) + + +def encode_projection(salt, N, K): + out = bytearray() + for tr in range(N // 32): + for tc in range(K // 256): + tile = bytearray(5120) + for g in range(8): + for r in range(32): + n, grp = tr * 32 + r, tc * 8 + g + struct.pack_into("