Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,11 @@ if(TARGET OpenMP::OpenMP_CXX)
target_link_libraries(mtp_draft_test PRIVATE OpenMP::OpenMP_CXX)
endif()
add_test(NAME mtp_draft COMMAND mtp_draft_test)
# engine#1781: the prompt-lookup drafter (npu/mtp/prompt_lookup.h), by hand and through mtp::run() on a toy base.
add_executable(prompt_lookup_test tests/prompt_lookup_test.cpp)
target_include_directories(prompt_lookup_test PRIVATE npu)
add_dependencies(onebit prompt_lookup_test)
add_test(NAME prompt_lookup COMMAND prompt_lookup_test)
# engine#1781 gate 2: the distribution-following acceptance rule (accept with min(1, p/q), resample the residual at the
# first rejection), with the temperature-0 identity and a deliberately wrong drafter whose output must still follow p.
# Header-only and device-free; pulled into CI through npu_sampling_test for the same reason as mtp_draft_test below.
Expand Down Expand Up @@ -2024,6 +2029,8 @@ endif()
# same sources against the runtime stub (engine#1946) cannot drift from this one.
onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt)
target_sources(onebit PRIVATE app/forward_serve.cpp)
# engine#1781: forward-serve --spec-draft runs a DFlash draft package (npu/dflash_head.h) as its drafter.
target_sources(onebit PRIVATE npu/dflash_forward.cpp npu/dflash_draft.cpp)
# engine#2111: the usage object's pure function, its own TU so tests/serve_usage_test.cpp can link it without
# dragging in npu/rt.h (see cmake/tests-serve-usage.cmake).
target_sources(onebit PRIVATE app/serve_usage.cpp)
Expand Down Expand Up @@ -2111,6 +2118,11 @@ endif()
set_tests_properties(gdn_batch_order PROPERTIES SKIP_RETURN_CODE 77)
set_tests_properties(layer_taps_model PROPERTIES SKIP_RETURN_CODE 77)
set_tests_properties(commit_full_accept PROPERTIES SKIP_RETURN_CODE 77)
# engine#1781: forward-serve --spec-lookup gives the plain loop's replies, end to end, host-GEMM on a container the
# script fabricates (no device, no model on disk). forward-serve exists only in this build, so it is a dev-box row.
add_test(NAME serve_spec_lookup
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/serve_spec_lookup_test.py $<TARGET_FILE:onebit>)
set_tests_properties(serve_spec_lookup PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 600)
set_tests_properties(batch_check_state PROPERTIES SKIP_RETURN_CODE 77)
# The fast-lane end-to-end test (npu_lane_e2e) is retired: its kernels were FastFlowLM-derived
# and its reference logits are lost. The NPU check is the private dx add-on's from-source
Expand Down
181 changes: 157 additions & 24 deletions app/forward_serve.cpp

Large diffs are not rendered by default.

29 changes: 29 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -3421,6 +3421,35 @@ fewer than `fallback_below` tokens on average, the run finishes with plain steps
round). `tools/dflash2_probe loop` with `DFLASH2_MTP_RUN=1` drives the served 27B through exactly these pieces, with
`DFlashHead` and the DFlash 2 selector as its pick hook, after the same batched prefill as its hand-written loop.

**Serving it: `forward-serve --spec-lookup K`** (off by default; `--spec-fallback X` adds the fallback). A request that
is greedy, has no grammar, and asks for no logprobs (and a server without `--dump-logits`) decodes through
`mtp::run_from()` with the prompt-lookup drafter (`npu/mtp/prompt_lookup.h`). That drafter needs no model: it proposes
the up to `K` tokens that followed the latest earlier occurrence of the stream's last one to three tokens. Every
other request takes the plain loop. The rule is that speculation is used only where it cannot change a token: each
token is still the tokenizer-bounded argmax, and both loops hand every token to the same code for end tokens, stop
sequences and the stop hold-back. The reply's last token is not run through the model (`Options::consume_last =
false`), as in the plain loop. The server logs one `[spec]` line per request (rounds, drafts accepted, plain steps).
`tests/serve_spec_lookup_test.py` (ctest `serve_spec_lookup`, on a build with `ONEBIT_NPU`) holds it to that end to
end. It fabricates a container, serves it host-GEMM twice (plain, then with `--spec-lookup 4`), and requires the same
text, `finish_reason` and `completion_tokens` for:
- every budget;
- stop sequences cut from the plain replies, so they do end them;
- a seeded sampled request, which must stay on the plain loop.

In host-GEMM mode the verify batch is 16 rows, the device's width. Host GEMMs have no width of their own, and the 1
the forward reports there would leave no row for a draft. The device route has not been measured with it yet.

**`forward-serve --spec-draft <dir>`** (`--spec-draft-k K`, default the package's block minus one) serves the same path
with a DFlash draft package as the drafter, in place of the prompt lookup. At load the package is decoded once and
checked against the target, before `ready`: the widths must match, and its target layers must exist. The forward then
writes those layers' hidden states into a tap sink on every pass. The sink is emptied when a request takes the engine,
so its rows start at the request's first position. `DFlashHead` drafts from it, picking with the target's own head:
DFlash 2's candidate selector when the package has one (`dflash2_select`, `npu/dflash_head.h`), else the bounded argmax.
A request that resumed from a saved prefix has no rows for the positions it skipped, so its head refuses to draft and
it decodes with plain steps: the same text, just no speedup. The same test serves a generation-1 package it writes
(`tests/dflash_draft_test.cpp`'s synthetic layout, sized to the fabricated target). On it every greedy request of two
or more tokens drafts, and every reply equals the plain server's.

**What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel
lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth
6 measured 0 %).
Expand Down
29 changes: 29 additions & 0 deletions npu/dflash_head.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,35 @@ struct HeadHooks {
std::function<int(const float* h, int prev)> pick;
};

// DFlash 2's candidate selector at one drafted row (the pick rule of a generation-2 package). `logits` is the target
// head on the row `h`; the top `selector_top_k` ids below `bound` (the tokenizer's size; 0 = all) are the candidates,
// each re-scored by selector_pick() with the pairwise term against `prev`. `hproj` is decode_plain(m.selector_hproj).
// Returns -1 when the package has no selector or there are fewer candidates than it wants.
inline int dflash2_select(const DraftModel& m, const std::vector<float>& hproj, const float* h, int prev,
const std::vector<float>& logits, int bound = 0) {
const int k = m.cfg.selector_top_k, R = m.cfg.selector_rank, H = m.cfg.hidden;
const int n = (bound > 0 && size_t(bound) < logits.size()) ? bound : int(logits.size());
if (k <= 0 || R <= 0 || n < k || hproj.size() != size_t(R) * size_t(H)) return -1;
std::vector<int> idx(static_cast<size_t>(n));
for (int j = 0; j < n; j++) idx[size_t(j)] = j;
std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) {
return logits[size_t(a)] > logits[size_t(b)] || (logits[size_t(a)] == logits[size_t(b)] && a < b);
});
std::vector<float> hp(static_cast<size_t>(R)), prow(static_cast<size_t>(R)), succ(static_cast<size_t>(k) * static_cast<size_t>(R)), unary(static_cast<size_t>(k));
for (int r = 0; r < R; r++) {
double acc = 0.0;
const float* wr = hproj.data() + size_t(r) * size_t(H);
for (int q = 0; q < H; q++) acc += double(wr[q]) * double(h[q]);
hp[size_t(r)] = float(acc);
}
decode_row(m.selector_pred, prev, prow.data());
for (int c = 0; c < k; c++) {
unary[size_t(c)] = logits[size_t(idx[size_t(c)])];
decode_row(m.selector_succ, idx[size_t(c)], succ.data() + size_t(c) * size_t(R));
}
return idx[size_t(selector_pick(unary.data(), k, prow.data(), hp.data(), succ.data(), R))];
}

class DFlashHead : public mtp::DraftHead {
public:
// `taps` is the forward's tap sink, one row of taps * hidden floats per position, starting at position 0.
Expand Down
23 changes: 14 additions & 9 deletions npu/mtp/draft.h
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,10 @@ struct Options {
// fallback_below <= 0 turns it off.
double fallback_below = 0.0;
int fallback_window = 3;
// When the budget ends on the base's own token, whether that token is still run through the base. true keeps the
// base holding every emitted token (what the tests and a caller that continues the stream want); false skips a
// pass whose logits nobody reads, which is what forward-serve does at the end of a reply.
bool consume_last = true;
};

struct Stats {
Expand Down Expand Up @@ -277,6 +281,7 @@ inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector<fl
if (t1 < 0) { r.err = "no logits"; return r; }
const bool keep_going = emit(t1);
if (!keep_going) break;
if (!opt.consume_last && (int)r.tokens.size() >= opt.max_tokens) break;

if (k <= 0 || fallback.off()) { // plain decode (no drafter, k = 0, or the fallback stopped it)
if (!base.step(t1, pos++, logits)) { r.err = "step failed"; return r; }
Expand Down Expand Up @@ -346,16 +351,16 @@ inline Result run_from(BaseModel& base, DraftHead* head, int pos, std::vector<fl
const std::vector<float>& rows = base.last_hidden_rows();
const size_t h = (size_t)base.hidden_size();
const size_t got = (h && rows.size() % h == 0) ? rows.size() / h : 0;
if (!rows.empty()) {
if (got != (size_t)(take + 1) && got != (size_t)(nd + 1)) {
r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " +
std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)";
return r;
}
std::vector<int> kept{t1};
kept.insert(kept.end(), d.begin(), d.begin() + take);
head->commit_rows(kept, rows, take + 1);
if (!rows.empty() && got != (size_t)(take + 1) && got != (size_t)(nd + 1)) {
r.err = "drafter rows: base returned " + std::to_string(got) + " rows, want " +
std::to_string(take + 1) + " (committed prefix) or " + std::to_string(nd + 1) + " (batch)";
return r;
}
// Always, even when the base keeps no rows (`rows` is then empty): a drafter that reads only the tokens
// (the prompt-lookup head) follows the stream through these calls alone.
std::vector<int> kept{t1};
kept.insert(kept.end(), d.begin(), d.begin() + take);
head->commit_rows(kept, rows, take + 1);
r.stats.ms_commit += lap(tp);
logits.assign(all.begin() + (size_t)take * (size_t)V, all.begin() + (size_t)(take + 1) * (size_t)V);
pos += take + 1;
Expand Down
78 changes: 78 additions & 0 deletions npu/mtp/prompt_lookup.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// prompt_lookup.h -- a drafter with no model: propose what followed the latest earlier occurrence of the stream's
// last tokens (engine#1781).
//
// The stream is the prompt plus every committed token, the last emitted token (t1) included. To draft, take its last n
// tokens (n from max_ngram down to min_ngram, the first length that matches wins), find their latest earlier
// occurrence, and propose the tokens that followed it. A reply that repeats its prompt -- code being edited, a quoted
// passage, a list that follows a pattern -- is then drafted at no cost but the verify batch, and a reply that does not
// gets no proposal at all, so the round falls back to a plain step. It never reads the base's hidden, so mtp::run()
// takes the one-pass round (needs_last_hidden() is false), and mtp::run's fallback bounds what a run of misses costs.
#pragma once

#include <algorithm>
#include <vector>

#include "draft.h"

namespace onebit::npu::mtp {

class PromptLookupHead : public DraftHead {
public:
explicit PromptLookupHead(std::vector<int> prompt, int max_ngram = 3, int min_ngram = 1)
: prompt_(std::move(prompt)), stream_(prompt_), max_n_(max_ngram), min_n_(min_ngram < 1 ? 1 : min_ngram) {}

bool needs_last_hidden() const override { return false; }

int draft(const std::vector<float>& /*hidden*/, int last_token, int pos, int k, std::vector<int>& out,
std::vector<float>* conf = nullptr) override {
out.clear();
if (conf) conf->clear(); // no probabilities: mtp::run skips its tau gate
// last_token runs at pos - 1, right after the committed stream; anything else means the stream is not the
// base's, and a proposal from it would be a guess about some other text.
if (k <= 0 || pos != int(stream_.size()) + 1) return 0;
stream_.push_back(last_token);
const int len = int(stream_.size());
for (int n = std::min(max_n_, len - 1); n >= min_n_; n--) {
const int* suf = stream_.data() + (len - n);
for (int i = len - n - 1; i >= 0; i--) { // latest earlier occurrence first
bool hit = true;
for (int j = 0; j < n && hit; j++) hit = stream_[size_t(i + j)] == suf[j];
if (!hit) continue;
for (int j = i + n; j < len && int(out.size()) < k; j++) out.push_back(stream_[size_t(j)]);
break;
}
if (!out.empty()) break;
}
stream_.pop_back();
return int(out.size());
}

void commit_rows(const std::vector<int>& tokens, const std::vector<float>& /*rows*/, int M) override {
for (int i = 0; i < M && i < int(tokens.size()); i++) stream_.push_back(tokens[size_t(i)]);
}

void reset() override { stream_ = prompt_; }

const std::vector<int>& stream() const { return stream_; }

private:
std::vector<int> prompt_, stream_;
int max_n_, min_n_;
};

} // namespace onebit::npu::mtp
19 changes: 19 additions & 0 deletions tests/mtp_draft_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1087,6 +1087,25 @@ int main() {
const Result re = onebit::npu::mtp::run(eq, &oh, P, f);
check(re.ok && same(re.tokens, ref.tokens) && re.stats.fell_back_at == -1,
"fallback: rounds of exactly 2 tokens against a bar of 2.0 keep drafting");
// consume_last = false: a run whose budget ends on the base's own token does not pay the pass after it.
for (int kind : {0, 1}) {
ToyBase m1(8), m2(8);
OnePassHead h1((OnePassHead::Kind)kind, (int)P.size()), h2((OnePassHead::Kind)kind, (int)P.size());
Options c = base_opt;
c.k = 4;
const Result a1 = onebit::npu::mtp::run(m1, &h1, P, c);
c.consume_last = false;
const Result a2 = onebit::npu::mtp::run(m2, &h2, P, c);
const int passes1 = m1.steps() + m1.batches(), passes2 = m2.steps() + m2.batches();
char msg[200];
std::snprintf(msg, sizeof msg, "consume_last=false, %s head: same text, %d passes against %d, base at %d of %d",
kind == 0 ? "perfect" : "garbage", passes2, passes1, m2.consumed(), m1.consumed());
// perfect at k=4 ends inside a batch (nothing to skip); garbage ends on the base's own token (one skipped)
const int skipped = kind == 0 ? 0 : 1;
check(a1.ok && a2.ok && same(a1.tokens, a2.tokens) && passes2 == passes1 - skipped &&
m2.consumed() == m1.consumed() - skipped,
msg);
}
}

std::printf(fails ? "\n%d CHECK(S) FAILED\n" : "\nall checks passed\n", fails);
Expand Down
Loading
Loading