diff --git a/docs/amd-drafters.md b/docs/amd-drafters.md
index 216c5a1f5..be2aca7c8 100644
--- a/docs/amd-drafters.md
+++ b/docs/amd-drafters.md
@@ -166,6 +166,29 @@ next step of engine#1780. What `dflash_forward_test` establishes:
- with a package directory (`-DONEBIT_DFLASH_DRAFT=
` or `ONEBIT_TEST_DFLASH_DRAFT`), the real draft decodes and
one pass over a made-up context is finite and position-independent.
+### DFlash 2 packages (engine#1781)
+
+`load_draft()` also reads `DFlash2DraftModel` packages (`FastFlowLM/Qwen3.8-27B-DFlash2-NPU2` is the one on this box),
+as generation 2 of the same loader. The config states the draft's interface under `dflash_config` and the rotation's base
+under `rope_parameters`. Every projection, the fusion matrix included, is a grid of 4736-byte Q4_K chunks rather than
+5120-byte q4_1 tiles. Each layer also carries two grouped convolutions, around the attention and around the MLP, and the
+package carries a candidate selector (a projection of the hidden state and two vocabulary-sized codebooks).
+`decode_draft()` fills the convolutions; the codebooks are read a row at a time with `decode_row()`, since each is
+254 MB as f32.
+
+The forward gains what generation 2 needs, each piece held to arithmetic written out in `dflash_forward_test`:
+
+| piece | what it computes |
+|---|---|
+| `draft_conv()` | one half of the grouped convolution: a row's output sums, over the kernel's offsets, (base kernel + the row's own kernel for its group) times the row that many places earlier |
+| `selector_pick()` | the selector's choice at one block row: among the target head's top candidates, the highest of logit + the pairwise term between the previous pick and the candidate |
+| `DraftKvCache`, `cache_extend()`, `draft_forward_cached()` | the context's keys and values per layer, computed once per context row and reused by every later pass; grown in steps it gives exactly the single-call pass |
+| `set_gemm_hook()` | an optional place for the draft's projections to run (the device route); unset, everything runs on the host |
+
+Measured with these pieces against the served 27B, in a local loop: on a code prompt, 6.86 tokens per round and 7.6-7.8
+tok/s against 4.2 tok/s plain decoding; output identical to plain greedy decoding. The loop is not part of this change;
+the numbers are on engine#1781.
+
## PARD-2's repos are weights-only — borrow the target's tokenizer
`amd/PARD2-*` ship **`README.md`, `config.json`, `model.safetensors`, `warp_model.bin`** and nothing else: **no tokenizer
diff --git a/npu/dflash_draft.cpp b/npu/dflash_draft.cpp
index 85a0cec8b..3c60bf7f8 100644
--- a/npu/dflash_draft.cpp
+++ b/npu/dflash_draft.cpp
@@ -65,6 +65,73 @@ float need_float(const json& c, const char* key, const std::string& path) {
bool close_to(double a, double b) { return std::fabs(a - b) <= 1e-5 * std::max(1.0, std::fabs(b)); }
+void read_target_layers(const json& from, const std::string& path, DraftConfig& o) {
+ if (!from.contains("target_layer_ids") || !from["target_layer_ids"].is_array() || from["target_layer_ids"].empty())
+ fail(path + ": no 'target_layer_ids'");
+ int prev = -1;
+ for (const auto& v : from["target_layer_ids"]) {
+ if (!v.is_number_integer() || v.get() < 0 || v.get() > 4096)
+ fail(path + ": 'target_layer_ids' holds something that is not a layer index");
+ const int id = int(v.get());
+ if (id <= prev) fail(path + ": 'target_layer_ids' is not strictly increasing at " + std::to_string(id));
+ o.target_layer_ids.push_back(id);
+ prev = id;
+ }
+}
+
+// DFlash2DraftModel: the geometry is at the top level as in any Qwen3 config, the draft's interface is under
+// "dflash_config", the rotation's base under "rope_parameters", and the weight file has the family's fixed name.
+DraftConfig read_generation2(const json& c, const std::string& path) {
+ DraftConfig o;
+ o.generation = 2;
+ o.hidden = need_int(c, "hidden_size", path);
+ o.intermediate = need_int(c, "intermediate_size", path);
+ o.layers = need_int(c, "num_hidden_layers", path);
+ o.heads = need_int(c, "num_attention_heads", path);
+ o.kv_heads = need_int(c, "num_key_value_heads", path);
+ o.head_dim = need_int(c, "head_dim", path);
+ o.vocab = need_int(c, "vocab_size", path);
+ o.sliding_window = need_int(c, "sliding_window", path);
+ o.rms_eps = need_float(c, "rms_norm_eps", path);
+ if (!c.contains("rope_parameters") || !c["rope_parameters"].is_object())
+ fail(path + ": no 'rope_parameters'");
+ o.rope_theta = need_float(c["rope_parameters"], "rope_theta", path + " rope_parameters");
+ o.embedding_scale = 1.0f;
+ if (!c.contains("dflash_config") || !c["dflash_config"].is_object()) fail(path + ": no 'dflash_config'");
+ const json& d = c["dflash_config"];
+ const std::string dp = path + " dflash_config";
+ o.block_size = need_int(d, "block_size", dp);
+ o.conv_kernel = need_int(d, "conv_kernel_size", dp);
+ o.conv_group = need_int(d, "conv_group_size", dp);
+ o.selector_rank = need_int(d, "selector_rank", dp);
+ o.selector_top_k = need_int(d, "selector_top_k", dp);
+ if (!d.contains("mask_token_id") || !d["mask_token_id"].is_number_integer() || d["mask_token_id"].get() < 0)
+ fail(dp + ": no usable 'mask_token_id'");
+ o.mask_token_id = d["mask_token_id"].get();
+ read_target_layers(d, dp, o);
+ if (c.contains("is_causal")) {
+ if (!c["is_causal"].is_boolean()) fail(path + ": 'is_causal' is not a boolean");
+ o.causal = c["is_causal"].get();
+ }
+ o.draft_weight = "dflash_weight.q4nx";
+ if (c.contains("draft_weight")) {
+ if (!c["draft_weight"].is_string() || c["draft_weight"].get().empty()) fail(path + ": 'draft_weight' is not a file name");
+ o.draft_weight = c["draft_weight"].get();
+ if (o.draft_weight.find('/') != std::string::npos || o.draft_weight.find("..") != std::string::npos)
+ fail(path + ": 'draft_weight' must be a file name inside the package, got '" + o.draft_weight + "'");
+ }
+ if (o.kv_heads > o.heads || o.heads % o.kv_heads != 0)
+ fail(path + ": " + std::to_string(o.heads) + " heads cannot share " + std::to_string(o.kv_heads) + " key/value heads");
+ if (o.block_size < 2) fail(path + ": block_size " + std::to_string(o.block_size) + " drafts nothing");
+ if (o.conv_kernel < 1 || o.conv_group < 1 || o.hidden % o.conv_group != 0)
+ fail(dp + ": a convolution of kernel " + std::to_string(o.conv_kernel) + " over groups of " +
+ std::to_string(o.conv_group) + " does not fit hidden_size " + std::to_string(o.hidden));
+ if (o.selector_rank < 1 || o.selector_top_k < 1 || o.selector_top_k > o.vocab)
+ fail(dp + ": selector_rank or selector_top_k is not usable");
+ if (o.causal) fail(path + ": a causal draft is not read by this loader");
+ return o;
+}
+
} // namespace
DraftConfig read_draft_config(const std::string& package_dir) {
@@ -79,15 +146,18 @@ DraftConfig read_draft_config(const std::string& package_dir) {
}
if (!c.is_object()) fail(path + ": not a JSON object");
const std::string type = c.value("model_type", json("")).is_string() ? c.value("model_type", std::string()) : "";
- if (type != "dflash") fail(path + ": model_type is '" + type + "', not 'dflash'");
- // The architecture decides which tensors exist. This loader knows one; a package naming another is a different
- // model (DFlash2 adds a selector and a convolution), and reading it with these rules would drop those silently.
+ // The architecture decides which tensors exist, so it is read first. Two are known; a package naming another is a
+ // different model, and reading it with these rules would drop what it adds silently.
+ std::string arch;
if (c.contains("architectures")) {
if (!c["architectures"].is_array() || c["architectures"].empty() || !c["architectures"][0].is_string())
fail(path + ": 'architectures' is not a list of names");
- const std::string arch = c["architectures"][0].get();
- if (arch != "DFlashDraftModel") fail(path + ": architecture '" + arch + "' is not read by this loader (DFlashDraftModel is)");
+ arch = c["architectures"][0].get();
}
+ if (arch == "DFlash2DraftModel") return read_generation2(c, path);
+ if (type != "dflash") fail(path + ": model_type is '" + type + "', not 'dflash'");
+ if (!arch.empty() && arch != "DFlashDraftModel")
+ fail(path + ": architecture '" + arch + "' is not read by this loader (DFlashDraftModel and DFlash2DraftModel are)");
DraftConfig o;
o.hidden = need_int(c, "hidden_size", path);
o.intermediate = need_int(c, "intermediate_size", path);
@@ -103,17 +173,7 @@ DraftConfig read_draft_config(const std::string& package_dir) {
if (!c.contains("mask_token_id") || !c["mask_token_id"].is_number_integer() || c["mask_token_id"].get() < 0)
fail(path + ": no usable 'mask_token_id'");
o.mask_token_id = c["mask_token_id"].get();
- if (!c.contains("target_layer_ids") || !c["target_layer_ids"].is_array() || c["target_layer_ids"].empty())
- fail(path + ": no 'target_layer_ids'");
- int prev = -1;
- for (const auto& v : c["target_layer_ids"]) {
- if (!v.is_number_integer() || v.get() < 0 || v.get() > 4096)
- fail(path + ": 'target_layer_ids' holds something that is not a layer index");
- const int id = int(v.get());
- if (id <= prev) fail(path + ": 'target_layer_ids' is not strictly increasing at " + std::to_string(id));
- o.target_layer_ids.push_back(id);
- prev = id;
- }
+ read_target_layers(c, path, o);
if (!c.contains("draft_weight") || !c["draft_weight"].is_string() || c["draft_weight"].get().empty())
fail(path + ": no 'draft_weight'");
o.draft_weight = c["draft_weight"].get();
@@ -172,13 +232,20 @@ DraftModel load_draft(const std::string& package_dir) {
v.cols = cols;
return v;
};
+ // A generation-2 package stores its projections as Q4_K chunks (4736 B per 32 x 256), not q4_1 tiles (5120 B).
+ const int chunk = c.generation == 2 ? Q4NX_CHUNK_Q4K : Q4NX_TILE_BYTES;
const auto projection = [&](const std::string& name, int n, int k) {
if (n % 32 != 0 || k % 256 != 0)
fail(path + ": '" + name + "' would be " + std::to_string(n) + " x " + std::to_string(k) +
", which is not a whole 32 x 256 tile grid");
- return take(name, "I8", q4nx_tensor_bytes(n, k), n, k);
+ TensorView v = take(name, "I8", size_t(n / 32) * size_t(k / 256) * size_t(chunk), n, k);
+ v.chunk = chunk;
+ return v;
};
const auto gain = [&](const std::string& name, int width) { return take(name, "BF16", size_t(width) * 2, 1, width); };
+ const auto plain = [&](const std::string& name, int rows, int cols) {
+ return take(name, "BF16", size_t(rows) * size_t(cols) * 2, rows, cols);
+ };
// The container repeats the interface scalars; a package whose two halves disagree is not one model.
const auto scalar = [&](const std::string& name, const char* dtype, size_t count) {
@@ -189,7 +256,7 @@ DraftModel load_draft(const std::string& package_dir) {
const auto agree = [&](bool same, const std::string& name, const std::string& got, const std::string& want) {
if (!same) fail(path + ": '" + name + "' is " + got + " and config.json says " + want);
};
- {
+ if (c.generation == 1) {
const uint8_t* p = scalar("dflash.block_size", "I32", 1);
agree(i32_at(p, 0) == c.block_size, "dflash.block_size", std::to_string(i32_at(p, 0)), std::to_string(c.block_size));
p = scalar("dflash.sliding_window", "I32", 1);
@@ -210,8 +277,15 @@ DraftModel load_draft(const std::string& package_dir) {
// and the declared shape must be that blocking -- a plain [hidden, taps*hidden] label would mean another element order.
if (c.hidden % 32 != 0) fail(path + ": hidden_size " + std::to_string(c.hidden) + " does not fill 32-row blocks");
const int fused = c.taps() * c.hidden;
- m.fc = take("model.fc.weight", "BF16", size_t(c.hidden) * size_t(fused) * 2, c.hidden, fused);
- {
+ if (c.generation == 2) {
+ m.fc = projection("model.fc.weight", c.hidden, fused);
+ const int groups = c.hidden / c.conv_group;
+ m.selector_hproj = plain("model.candidate_selector.hidden_projection.weight", c.selector_rank, c.hidden);
+ m.selector_pred = plain("model.candidate_selector.predecessor_codebook", c.vocab, c.selector_rank);
+ m.selector_succ = plain("model.candidate_selector.successor_codebook", c.vocab, c.selector_rank);
+ (void)groups;
+ } else {
+ m.fc = take("model.fc.weight", "BF16", size_t(c.hidden) * size_t(fused) * 2, c.hidden, fused);
const std::vector want = {c.hidden / 32, fused, 32};
if (by_name["model.fc.weight"]->shape != want)
fail(path + ": 'model.fc.weight' is not declared [hidden/32, taps*hidden, 32], so its element order is not the one this loader undoes");
@@ -232,11 +306,18 @@ DraftModel load_draft(const std::string& package_dir) {
L.post_norm = gain(p + "post_attention_layernorm.weight", c.hidden);
L.q_norm = gain(p + "self_attn.q_norm.weight", c.head_dim);
L.k_norm = gain(p + "self_attn.k_norm.weight", c.head_dim);
+ if (c.generation == 2) {
+ const int kernels = 2 * c.conv_kernel, groups = c.hidden / c.conv_group;
+ L.attn_conv_base = plain(p + "attention_conv.base_kernel", kernels, c.hidden);
+ L.attn_conv_proj = plain(p + "attention_conv.kernel_projection.weight", kernels * groups, c.hidden);
+ L.mlp_conv_base = plain(p + "mlp_conv.base_kernel", kernels, c.hidden);
+ L.mlp_conv_proj = plain(p + "mlp_conv.kernel_projection.weight", kernels * groups, c.hidden);
+ }
m.layers.push_back(L);
}
for (const auto& t : table.tensors)
if (used.count(t.name) == 0)
- fail(path + ": tensor '" + t.name + "' is not part of a DFlashDraftModel (" + std::to_string(table.tensors.size() - used.size()) +
+ fail(path + ": tensor '" + t.name + "' is not part of this draft's architecture (" + std::to_string(table.tensors.size() - used.size()) +
" unexplained): this is another architecture");
return m;
}
@@ -246,9 +327,11 @@ std::vector decode_projection(const TensorView& t) {
std::vector out(size_t(t.rows) * size_t(t.cols));
// Not the size-dispatching q4nx_dequant_tensor(): a projection here is a tile grid by contract, and a grid whose
// byte count happened to equal rows*cols*2 must not be read as plain 2-byte floats.
- if (t.bytes != q4nx_tensor_bytes(t.rows, t.cols) ||
- q4nx_dequant_tensor_chunked(t.data, t.bytes, t.rows, t.cols, 5120, out.data()) != 0)
- fail("'" + t.name + "' does not decode as a " + std::to_string(t.rows) + " x " + std::to_string(t.cols) + " q4_1 tile grid");
+ const int chunk = t.chunk > 0 ? t.chunk : Q4NX_TILE_BYTES;
+ if (t.rows % 32 != 0 || t.cols % 256 != 0 || t.bytes != size_t(t.rows / 32) * size_t(t.cols / 256) * size_t(chunk) ||
+ q4nx_dequant_tensor_chunked(t.data, t.bytes, t.rows, t.cols, chunk, out.data()) != 0)
+ fail("'" + t.name + "' does not decode as a " + std::to_string(t.rows) + " x " + std::to_string(t.cols) + " grid of " +
+ std::to_string(chunk) + "-byte tiles");
return out;
}
@@ -259,6 +342,20 @@ std::vector decode_gain(const TensorView& t) {
return out;
}
+std::vector decode_plain(const TensorView& t) {
+ if (!t.ok() || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) fail("decode_plain of '" + t.name + "': not 2 bytes per element");
+ std::vector out(size_t(t.rows) * size_t(t.cols));
+ for (size_t i = 0; i < out.size(); ++i) out[i] = bf16(t.data + i * 2);
+ return out;
+}
+
+void decode_row(const TensorView& t, int row, float* out) {
+ if (!t.ok() || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) fail("decode_row of '" + t.name + "': not 2 bytes per element");
+ if (row < 0 || row >= t.rows || out == nullptr) fail("decode_row of '" + t.name + "': row " + std::to_string(row) + " is outside the tensor");
+ const uint8_t* p = t.data + size_t(row) * size_t(t.cols) * 2;
+ for (int i = 0; i < t.cols; ++i) out[i] = bf16(p + size_t(i) * 2);
+}
+
std::vector decode_fusion(const TensorView& t) {
if (!t.ok() || t.rows % 32 != 0 || t.bytes != size_t(t.rows) * size_t(t.cols) * 2)
fail("decode_fusion of '" + t.name + "': not hidden x fused at 2 bytes per element");
diff --git a/npu/dflash_draft.h b/npu/dflash_draft.h
index d5e72ad51..42831732b 100644
--- a/npu/dflash_draft.h
+++ b/npu/dflash_draft.h
@@ -59,6 +59,13 @@ struct DraftConfig {
float rope_theta = 0.0f;
float embedding_scale = 0.0f;
bool causal = false;
+ // Which draft this is. 1: DFlashDraftModel (model_type "dflash"). 2: DFlash2DraftModel, which adds a grouped
+ // convolution around each layer's attention and MLP and a candidate selector, states its interface under
+ // "dflash_config", and stores every projection (the fusion matrix included) as Q4_K chunks.
+ int generation = 1;
+ int conv_kernel = 0, conv_group = 0; // generation 2
+ int selector_rank = 0, selector_top_k = 0; // generation 2
+ int vocab = 0; // generation 2: rows of the selector's codebooks
std::string draft_weight; // the container's file name, relative to the package
std::string base_model; // the target this draft was made for, as the package names it (may be empty)
@@ -74,17 +81,25 @@ struct TensorView {
size_t bytes = 0;
int rows = 0; // N (out); 1 for a vector
int cols = 0; // K (in), or the vector's length
+ int chunk = 0; // a projection's bytes per 32 x 256 tile: 0 or 5120 = q4_1, 4736 = Q4_K
bool ok() const { return data != nullptr && bytes != 0; }
};
struct DraftLayer {
TensorView q, k, v, o, gate, up, down; // q4_1 tile grids, N x K
TensorView input_norm, post_norm, q_norm, k_norm; // 2-byte gains
+ // Generation 2, 2 bytes per element: base kernels [2][kernel][hidden], kernel projections
+ // [2 * kernel * hidden / group][hidden] row-major.
+ TensorView attn_conv_base, attn_conv_proj, mlp_conv_base, mlp_conv_proj;
};
struct DraftModel {
DraftConfig cfg;
- TensorView fc; // hidden x (taps * hidden): fuses the target layers' hidden states
+ TensorView fc; // hidden x (taps * hidden): fuses the target layers' hidden states (generation 1: 2-byte
+ // elements in 32-row blocks, decode_fusion(); generation 2: a tile grid, decode_projection())
+ // Generation 2: the candidate selector. hidden_projection [rank][hidden], the two codebooks [vocab][rank], all at
+ // 2 bytes per element. The codebooks are 254 MB each as f32, so a caller reads rows out of the view (decode_row()).
+ TensorView selector_hproj, selector_pred, selector_succ;
TensorView hidden_norm; // applied to the fused features
TensorView norm; // the final norm
std::vector layers;
@@ -105,6 +120,12 @@ std::vector decode_projection(const TensorView& t);
// A norm's gains as f32.
std::vector decode_gain(const TensorView& t);
+// A plain 2-byte-per-element tensor as f32, row-major [rows][cols].
+std::vector decode_plain(const TensorView& t);
+
+// One row of a plain 2-byte-per-element tensor into out[cols]. Throws if the row is outside the tensor.
+void decode_row(const TensorView& t, int row, float* out);
+
// The fusion matrix as f32, row-major [hidden][taps * hidden]. The container stores it as [hidden/32][taps*hidden][32]
// -- 32-row blocks, column by column, the row inside the block last -- and that is the order this undoes:
// out[(a*32 + r) * cols + k] = stored[(a * cols + k) * 32 + r].
diff --git a/npu/dflash_forward.cpp b/npu/dflash_forward.cpp
index a9c56a9e6..5d730498b 100644
--- a/npu/dflash_forward.cpp
+++ b/npu/dflash_forward.cpp
@@ -31,14 +31,22 @@ void need(bool ok, const std::string& what) {
if (!ok) throw std::runtime_error("dflash forward: " + what);
}
+GemmHook g_hook = nullptr;
+void* g_hook_user = nullptr;
+
// y[rows][n_out] = x[rows][n_in] * W^T, W row-major [n_out][n_in].
std::vector linear(const std::vector& W, int n_out, int n_in, const float* x, int rows) {
std::vector y(size_t(rows) * size_t(n_out));
- for (int r = 0; r < rows; r++) {
- const float* xr = x + size_t(r) * size_t(n_in);
- for (int n = 0; n < n_out; n++) {
- const float* wr = W.data() + size_t(n) * size_t(n_in);
+ if (g_hook && rows > 0 && g_hook(W.data(), n_out, n_in, x, rows, y.data(), g_hook_user)) return y;
+ // Output rows are independent, so they are shared out across threads when the build has OpenMP; the sum itself
+ // stays in double and may be taken in vector lanes, so two builds can differ in the last bits of a result.
+ #pragma omp parallel for schedule(static)
+ for (int n = 0; n < n_out; n++) {
+ const float* wr = W.data() + size_t(n) * size_t(n_in);
+ for (int r = 0; r < rows; r++) {
+ const float* xr = x + size_t(r) * size_t(n_in);
double acc = 0.0;
+ #pragma omp simd reduction(+ : acc)
for (int k = 0; k < n_in; k++) acc += double(wr[k]) * double(xr[k]);
y[size_t(r) * size_t(n_out) + size_t(n)] = float(acc);
}
@@ -69,8 +77,76 @@ void rope(float* v, int d, long pos, float theta) {
}
}
+// One half of a DFlash 2 convolution over `rows` rows of H. `dyn` is [rows][kernel][groups] for this half.
+std::vector convolve(const float* x, int rows, int H, int kernel, int group, const float* base, const float* dyn) {
+ const int groups = H / group;
+ std::vector out(size_t(rows) * size_t(H), 0.0f);
+ for (int t = 0; t < rows; t++)
+ for (int off = 0; off < kernel && off <= t; off++) {
+ const float* src = x + size_t(t - off) * size_t(H);
+ const float* b = base + size_t(off) * size_t(H);
+ const float* d = dyn + (size_t(t) * size_t(kernel) + size_t(off)) * size_t(groups);
+ float* o = out.data() + size_t(t) * size_t(H);
+ for (int g = 0; g < groups; g++)
+ for (int j = 0; j < group; j++) {
+ const int i = g * group + j;
+ o[i] += (b[i] + d[g]) * src[i];
+ }
+ }
+ return out;
+}
+
+// prepare(): the row's kernels from the row, the first half applied now, the second half kept for finish().
+std::vector conv_prepare(const DraftWeights& w, const DraftConv& c, const std::vector& x, int rows,
+ std::vector& keep) {
+ const int H = w.hidden, K = w.conv_kernel, G = H / w.conv_group;
+ const std::vector dyn = linear(c.proj, 2 * K * G, H, x.data(), rows); // [rows][2][K][G]
+ std::vector first(size_t(rows) * size_t(K) * size_t(G));
+ keep.assign(first.size(), 0.0f);
+ for (int t = 0; t < rows; t++) {
+ const float* d = dyn.data() + size_t(t) * size_t(2 * K * G);
+ std::copy(d, d + K * G, first.begin() + ptrdiff_t(size_t(t) * size_t(K * G)));
+ std::copy(d + K * G, d + 2 * K * G, keep.begin() + ptrdiff_t(size_t(t) * size_t(K * G)));
+ }
+ return convolve(x.data(), rows, H, K, w.conv_group, c.base.data(), first.data());
+}
+
+std::vector conv_finish(const DraftWeights& w, const DraftConv& c, const std::vector& x, int rows,
+ const std::vector& keep) {
+ return convolve(x.data(), rows, w.hidden, w.conv_kernel, w.conv_group,
+ c.base.data() + size_t(w.conv_kernel) * size_t(w.hidden), keep.data());
+}
+
} // namespace
+std::vector draft_conv(const float* x, int rows, int hidden, int kernel, int group, const float* base,
+ const float* dyn) {
+ need(x && base && dyn && rows > 0 && hidden > 0 && kernel > 0 && group > 0 && hidden % group == 0,
+ "a convolution with an empty input, or a group size that does not divide the hidden width");
+ return convolve(x, rows, hidden, kernel, group, base, dyn);
+}
+
+int selector_pick(const float* unary, int k, const float* prev, const float* h, const float* succ, int rank) {
+ need(unary && prev && h && succ && k > 0 && rank > 0, "a selector row with no candidates or no codebook rows");
+ int best = 0;
+ double top = 0.0;
+ for (int c = 0; c < k; c++) {
+ double score = unary[c];
+ const float* s = succ + size_t(c) * size_t(rank);
+ for (int r = 0; r < rank; r++) score += double(prev[r]) * double(h[r]) * double(s[r]);
+ if (c == 0 || score > top) {
+ top = score;
+ best = c;
+ }
+ }
+ return best;
+}
+
+void set_gemm_hook(GemmHook hook, void* user) {
+ g_hook = hook;
+ g_hook_user = user;
+}
+
void check_weights(const DraftWeights& w) {
need(w.hidden > 0 && w.intermediate > 0 && w.heads > 0 && w.kv_heads > 0 && w.head_dim > 0 && w.taps > 0,
"a geometry field is zero or negative");
@@ -94,6 +170,18 @@ void check_weights(const DraftWeights& w) {
need(L.input_norm.size() == H && L.post_norm.size() == H, at + "a layer norm is not hidden wide");
need(L.q_norm.size() == size_t(w.head_dim) && L.k_norm.size() == size_t(w.head_dim),
at + "q_norm or k_norm is not head_dim wide");
+ for (const DraftConv* c : {&L.attn_conv, &L.mlp_conv}) {
+ if (w.conv_kernel == 0) {
+ need(!c->present(), at + "a convolution on a draft that declares none");
+ continue;
+ }
+ need(w.conv_kernel > 0 && w.conv_group > 0 && w.hidden % w.conv_group == 0,
+ "the convolution's kernel or group size does not fit the hidden width");
+ const size_t G = size_t(w.hidden / w.conv_group);
+ need(c->base.size() == 2 * size_t(w.conv_kernel) * H, at + "a convolution's base kernel is not 2 x kernel x hidden");
+ need(c->proj.size() == 2 * size_t(w.conv_kernel) * G * H,
+ at + "a convolution's kernel projection is not (2 * kernel * groups) x hidden");
+ }
}
}
@@ -107,7 +195,9 @@ DraftWeights decode_draft(const DraftModel& m) {
w.taps = m.cfg.taps();
w.rms_eps = m.cfg.rms_eps;
w.rope_theta = m.cfg.rope_theta;
- w.fc = decode_fusion(m.fc);
+ w.fc = m.cfg.generation == 2 ? decode_projection(m.fc) : decode_fusion(m.fc);
+ w.conv_kernel = m.cfg.conv_kernel;
+ w.conv_group = m.cfg.conv_group;
w.hidden_norm = decode_gain(m.hidden_norm);
w.norm = decode_gain(m.norm);
w.layers.reserve(m.layers.size());
@@ -124,53 +214,113 @@ DraftWeights decode_draft(const DraftModel& m) {
d.post_norm = decode_gain(L.post_norm);
d.q_norm = decode_gain(L.q_norm);
d.k_norm = decode_gain(L.k_norm);
+ if (m.cfg.generation == 2) {
+ d.attn_conv.base = decode_plain(L.attn_conv_base);
+ d.attn_conv.proj = decode_plain(L.attn_conv_proj);
+ d.mlp_conv.base = decode_plain(L.mlp_conv_base);
+ d.mlp_conv.proj = decode_plain(L.mlp_conv_proj);
+ }
w.layers.push_back(std::move(d));
}
check_weights(w);
return w;
}
+std::vector fuse_context(const DraftWeights& w, const float* taps, int ctx) {
+ check_weights(w);
+ need(ctx >= 0 && (ctx == 0 || taps != nullptr), "a context length without context rows");
+ std::vector c;
+ if (ctx > 0) {
+ c = linear(w.fc, w.hidden, w.hidden * w.taps, taps, ctx);
+ rms_norm(c.data(), size_t(ctx), w.hidden, w.hidden_norm, w.rms_eps);
+ }
+ return c;
+}
+
std::vector draft_forward(const DraftWeights& w, const float* taps, int ctx, const float* noise, int block,
long pos0, int window) {
+ const std::vector c = fuse_context(w, taps, ctx);
+ return draft_forward_fused(w, c.data(), ctx, noise, block, pos0, window);
+}
+
+void cache_extend(const DraftWeights& w, DraftKvCache& c, const float* fused_rows, int n) {
+ check_weights(w);
+ need(n >= 0 && (n == 0 || fused_rows != nullptr), "context rows to add without the rows");
+ const int H = w.hidden, D = w.head_dim, KV = w.kv_heads * D;
+ if (c.k.size() != w.layers.size()) {
+ need(c.rows == 0, "a cache whose layer count does not match the draft");
+ c.k.assign(w.layers.size(), {});
+ c.v.assign(w.layers.size(), {});
+ }
+ if (n == 0) return;
+ for (size_t l = 0; l < w.layers.size(); l++) {
+ const DraftLayerWeights& L = w.layers[l];
+ std::vector k = linear(L.k, KV, H, fused_rows, n);
+ const std::vector v = linear(L.v, KV, H, fused_rows, n);
+ rms_norm(k.data(), size_t(n) * size_t(w.kv_heads), D, L.k_norm, w.rms_eps);
+ for (int t = 0; t < n; t++)
+ for (int hd = 0; hd < w.kv_heads; hd++)
+ rope(k.data() + size_t(t) * size_t(KV) + size_t(hd) * size_t(D), D, c.pos0 + c.rows + t, w.rope_theta);
+ c.k[l].insert(c.k[l].end(), k.begin(), k.end());
+ c.v[l].insert(c.v[l].end(), v.begin(), v.end());
+ }
+ c.rows += n;
+}
+
+std::vector draft_forward_fused(const DraftWeights& w, const float* fused, int ctx, const float* noise, int block,
+ long pos0, int window) {
+ need(ctx >= 0 && (ctx == 0 || fused != nullptr), "a context length without context rows");
+ DraftKvCache c;
+ c.pos0 = pos0;
+ cache_extend(w, c, fused, ctx);
+ return draft_forward_cached(w, c, noise, block, window);
+}
+
+std::vector draft_forward_cached(const DraftWeights& w, const DraftKvCache& c, const float* noise, int block,
+ int window) {
check_weights(w);
need(block > 0 && noise != nullptr, "an empty block");
- need(ctx >= 0 && (ctx == 0 || taps != nullptr), "a context length without context rows");
need(window >= 0, "a negative window");
+ need(c.rows == 0 || c.k.size() == w.layers.size(), "a cache that was not built for this draft");
const int H = w.hidden, I = w.intermediate, D = w.head_dim;
const int Q = w.heads * D, KV = w.kv_heads * D;
+ const int ctx = c.rows;
+ const long pos0 = c.pos0;
const int keys = ctx + block;
const int group = w.heads / w.kv_heads; // query heads per key/value head, consecutive
- // The context: fused, normalised once, and the same rows for every layer.
- std::vector c;
- if (ctx > 0) {
- c = linear(w.fc, H, H * w.taps, taps, ctx);
- rms_norm(c.data(), size_t(ctx), H, w.hidden_norm, w.rms_eps);
- }
-
+ // The context's keys and values come from the cache: they depend only on the fused context rows and the layer,
+ // never on the block, so a caller that drafts many blocks over one growing context computes each row once.
std::vector x(noise, noise + size_t(block) * size_t(H));
- std::vector kv_in(size_t(keys) * size_t(H));
- std::copy(c.begin(), c.end(), kv_in.begin());
std::vector score(static_cast(keys));
std::vector attn(size_t(block) * size_t(Q));
- for (const DraftLayerWeights& L : w.layers) {
+ for (size_t l = 0; l < w.layers.size(); l++) {
+ const DraftLayerWeights& L = w.layers[l];
// attention
std::vector h = x;
rms_norm(h.data(), size_t(block), H, L.input_norm, w.rms_eps);
- std::copy(h.begin(), h.end(), kv_in.begin() + ptrdiff_t(size_t(ctx) * size_t(H)));
+ std::vector attn_keep;
+ if (L.attn_conv.present()) h = conv_prepare(w, L.attn_conv, h, block, attn_keep);
std::vector q = linear(L.q, Q, H, h.data(), block);
- std::vector k = linear(L.k, KV, H, kv_in.data(), keys);
- std::vector v = linear(L.v, KV, H, kv_in.data(), keys);
+ std::vector kb = linear(L.k, KV, H, h.data(), block);
+ const std::vector vb = linear(L.v, KV, H, h.data(), block);
rms_norm(q.data(), size_t(block) * size_t(w.heads), D, L.q_norm, w.rms_eps);
- rms_norm(k.data(), size_t(keys) * size_t(w.kv_heads), D, L.k_norm, w.rms_eps);
- for (int i = 0; i < block; i++)
+ rms_norm(kb.data(), size_t(block) * size_t(w.kv_heads), D, L.k_norm, w.rms_eps);
+ for (int i = 0; i < block; i++) {
for (int hd = 0; hd < w.heads; hd++)
rope(q.data() + size_t(i) * size_t(Q) + size_t(hd) * size_t(D), D, pos0 + ctx + i, w.rope_theta);
- for (int t = 0; t < keys; t++)
for (int hd = 0; hd < w.kv_heads; hd++)
- rope(k.data() + size_t(t) * size_t(KV) + size_t(hd) * size_t(D), D, pos0 + t, w.rope_theta);
+ rope(kb.data() + size_t(i) * size_t(KV) + size_t(hd) * size_t(D), D, pos0 + ctx + i, w.rope_theta);
+ }
+ // keys and values: the cached context rows, then the block's own
+ const auto key_at = [&](int t) {
+ return t < ctx ? c.k[l].data() + size_t(t) * size_t(KV) : kb.data() + size_t(t - ctx) * size_t(KV);
+ };
+ const auto val_at = [&](int t) {
+ return t < ctx ? c.v[l].data() + size_t(t) * size_t(KV) : vb.data() + size_t(t - ctx) * size_t(KV);
+ };
const double scale = 1.0 / std::sqrt(double(D));
for (int i = 0; i < block; i++) {
@@ -184,7 +334,7 @@ std::vector draft_forward(const DraftWeights& w, const float* taps, int c
score[size_t(t)] = -INFINITY;
continue;
}
- const float* kvp = k.data() + size_t(t) * size_t(KV) + size_t(kh) * size_t(D);
+ const float* kvp = key_at(t) + size_t(kh) * size_t(D);
double s = 0.0;
for (int d = 0; d < D; d++) s += double(qv[d]) * double(kvp[d]);
score[size_t(t)] = s * scale;
@@ -199,18 +349,20 @@ std::vector draft_forward(const DraftWeights& w, const float* taps, int c
float* out = attn.data() + size_t(i) * size_t(Q) + size_t(hd) * size_t(D);
for (int d = 0; d < D; d++) {
double acc = 0.0;
- for (int t = 0; t < keys; t++)
- acc += score[size_t(t)] * double(v[size_t(t) * size_t(KV) + size_t(kh) * size_t(D) + size_t(d)]);
+ for (int t = 0; t < keys; t++) acc += score[size_t(t)] * double(val_at(t)[size_t(kh) * size_t(D) + size_t(d)]);
out[d] = float(acc / sum);
}
}
}
std::vector o = linear(L.o, H, Q, attn.data(), block);
+ if (L.attn_conv.present()) o = conv_finish(w, L.attn_conv, o, block, attn_keep);
for (size_t j = 0; j < x.size(); j++) x[j] += o[j];
// MLP
std::vector p = x;
rms_norm(p.data(), size_t(block), H, L.post_norm, w.rms_eps);
+ std::vector mlp_keep;
+ if (L.mlp_conv.present()) p = conv_prepare(w, L.mlp_conv, p, block, mlp_keep);
std::vector g = linear(L.gate, I, H, p.data(), block);
std::vector u = linear(L.up, I, H, p.data(), block);
for (size_t j = 0; j < g.size(); j++) {
@@ -218,6 +370,7 @@ std::vector draft_forward(const DraftWeights& w, const float* taps, int c
g[j] = float(a / (1.0 + std::exp(-a)) * double(u[j])); // silu(gate) * up
}
std::vector dn = linear(L.down, H, I, g.data(), block);
+ if (L.mlp_conv.present()) dn = conv_finish(w, L.mlp_conv, dn, block, mlp_keep);
for (size_t j = 0; j < x.size(); j++) x[j] += dn[j];
}
diff --git a/npu/dflash_forward.h b/npu/dflash_forward.h
index d739c85c9..39431a27b 100644
--- a/npu/dflash_forward.h
+++ b/npu/dflash_forward.h
@@ -44,21 +44,40 @@
namespace onebit::npu::dflash {
+// DFlash 2 only: a grouped convolution over the block rows, applied before and after the attention and before and
+// after the MLP. The hidden vector is cut into groups of `group` values. A row's kernel is a fixed part (`base`) plus
+// a part computed from the row itself (`proj` of the row, one value per offset and group), and the output at row t
+// sums, over offset 0..kernel-1, (base + dynamic) * x[t - offset]; rows before the block count as zero. `prepare`
+// uses the first half of `base` and of the projection's output, `finish` the second half.
+struct DraftConv {
+ std::vector base; // [2][kernel][hidden]
+ std::vector proj; // row-major [2 * kernel * (hidden / group)][hidden]
+ bool present() const { return !base.empty(); }
+};
+
struct DraftLayerWeights {
std::vector q, k, v, o, gate, up, down; // row-major [out][in]
std::vector input_norm, post_norm, q_norm, k_norm; // gains
+ DraftConv attn_conv, mlp_conv; // empty on a first-generation draft
};
// A draft decoded to f32. For the Bolt draft this is about 1.3 GB; the caller decides how long to keep it.
struct DraftWeights {
int hidden = 0, intermediate = 0, heads = 0, kv_heads = 0, head_dim = 0, taps = 0;
float rms_eps = 0.0f, rope_theta = 0.0f;
+ int conv_kernel = 0, conv_group = 0; // DFlash 2; 0 when the layers carry no convolution
std::vector fc; // [hidden][taps * hidden]
std::vector hidden_norm; // [hidden]
std::vector norm; // [hidden]
std::vector layers;
};
+// An optional place for the projections to run. When set, every matrix product of the draft forward is offered to
+// it first: W is the row-major [n_out][n_in] matrix the product would use (its address identifies the projection),
+// x is [rows][n_in], y is [rows][n_out]. Return false to have the host compute it instead. One hook per process.
+using GemmHook = bool (*)(const float* W, int n_out, int n_in, const float* x, int rows, float* y, void* user);
+void set_gemm_hook(GemmHook hook, void* user);
+
// Decode every tensor of a loaded package. Throws std::runtime_error if a decoded tensor has the wrong size.
DraftWeights decode_draft(const DraftModel& m);
@@ -75,4 +94,38 @@ void check_weights(const DraftWeights& w);
std::vector draft_forward(const DraftWeights& w, const float* taps, int ctx, const float* noise, int block,
long pos0 = 0, int window = 0);
+// One half of a DFlash 2 convolution, on its own so it can be held to a hand-computed case: `rows` rows of
+// `hidden`, `base` is [kernel][hidden] and `dyn` is [rows][kernel][hidden / group] (a row's own kernel, per offset
+// and group). out[t][i] = sum over offset < kernel, offset <= t of (base[offset][i] + dyn[t][offset][i / group]) *
+// x[t - offset][i]. Rows before the first count as zero.
+std::vector draft_conv(const float* x, int rows, int hidden, int kernel, int group, const float* base,
+ const float* dyn);
+
+// The DFlash 2 candidate selector's choice at one block row. `unary` holds the target head's logits of the `k`
+// candidates; `prev` is the predecessor codebook's row of the token picked for the row before (the last accepted
+// token for the first row), `h` the selector's projection of this row's hidden state, both `rank` wide; `succ` is
+// the successor codebook's rows of the candidates, [k][rank]. A candidate's score is
+// unary + sum_r prev[r] * h[r] * succ[r]; the highest wins and the first of equals is kept. Returns its index.
+int selector_pick(const float* unary, int k, const float* prev, const float* h, const float* succ, int rank);
+
+// The context's keys and values per layer, for a caller that drafts many blocks over one growing context: they depend
+// only on the fused context rows, so each row is projected, normalised and rotated once (at position pos0 + its
+// index) and reused by every later pass. draft_forward_fused() builds one for its own call.
+struct DraftKvCache {
+ long pos0 = 0;
+ int rows = 0;
+ std::vector> k, v; // per layer, [rows][kv_heads * head_dim]
+};
+// Append `n` fused context rows ([n][hidden], from fuse_context()) to the cache.
+void cache_extend(const DraftWeights& w, DraftKvCache& c, const float* fused_rows, int n);
+// One pass over the cache's context rows (positions pos0 .. pos0 + rows - 1) and a block placed after them.
+std::vector draft_forward_cached(const DraftWeights& w, const DraftKvCache& c, const float* noise, int block,
+ int window = 0);
+
+// The same pass in two steps, for a caller that drafts many blocks over one growing context: the context rows are
+// fused and normalised once ([ctx][hidden]) and handed to every pass that needs a prefix of them.
+std::vector fuse_context(const DraftWeights& w, const float* taps, int ctx);
+std::vector draft_forward_fused(const DraftWeights& w, const float* fused, int ctx, const float* noise, int block,
+ long pos0 = 0, int window = 0);
+
} // namespace onebit::npu::dflash
diff --git a/tests/dflash_draft_test.cpp b/tests/dflash_draft_test.cpp
index 52735cd82..62fdc83cb 100644
--- a/tests/dflash_draft_test.cpp
+++ b/tests/dflash_draft_test.cpp
@@ -298,7 +298,7 @@ void test_synthetic(const fs::path& root) {
refused(with("short", [&](std::vector& t) { find(t, "model.layers.0.self_attn.k_proj.weight").bytes.resize(5120); }, cfg),
"'model.layers.0.self_attn.k_proj.weight' holds 5120 B", "a projection with too few tiles");
refused(with("extra", [&](std::vector& t) { t.push_back({"model.selector.weight", "BF16", "[4]", std::string(8, '\0')}); }, cfg),
- "'model.selector.weight' is not part of a DFlashDraftModel", "a tensor of another architecture");
+ "'model.selector.weight' is not part of this draft's architecture", "a tensor of another architecture");
refused(with("dtype", [&](std::vector& t) { find(t, "model.norm.weight").dtype = "F16"; }, cfg),
"'model.norm.weight' is declared F16", "a gain under another dtype label");
refused(with("plainfc", [&](std::vector& t) { find(t, "model.fc.weight").shape = "[256,512]"; }, cfg),
@@ -308,7 +308,9 @@ void test_synthetic(const fs::path& root) {
refused(with("taps", [&](std::vector& t) { const int32_t v = 2; std::memcpy(find(t, "dflash.target_layer_ids").bytes.data() + 4, &v, 4); }, cfg),
"'dflash.target_layer_ids[1]' is 2 and config.json says 3", "a container that disagrees on a target layer");
refused(with("dflash2", none, replaced(cfg, "DFlashDraftModel", "DFlash2DraftModel")),
- "architecture 'DFlash2DraftModel' is not read by this loader", "a DFlash2 package");
+ "no 'vocab_size'", "a first-generation config relabelled DFlash2 is read by the DFlash2 rules, and refused by name");
+ refused(with("dflash3", none, replaced(cfg, "DFlashDraftModel", "DFlash3DraftModel")),
+ "architecture 'DFlash3DraftModel' is not read by this loader", "an architecture this loader does not know");
refused(with("type", none, replaced(cfg, "\"model_type\":\"dflash\"", "\"model_type\":\"qwen3\"")),
"model_type is 'qwen3'", "a config that is not a draft's");
refused(with("nokey", none, replaced(cfg, "\"head_dim\":64,", "")), "no 'head_dim'", "a config without head_dim");
@@ -423,6 +425,43 @@ void test_real(const std::string& dir) {
} // namespace
+// A DFlash 2 package (FastFlowLM/Qwen3.8-27B-DFlash2-NPU2 on this box): every tensor accounted for, one projection of
+// each kind decodes finite, and the selector's codebooks give a row each. Decoding the whole draft is 6 GB as f32, so
+// this reads one of each instead.
+void test_real2(const std::string& dir) {
+ DraftModel m;
+ try {
+ m = load_draft(dir);
+ } catch (const std::exception& e) {
+ check(false, std::string("the DFlash 2 package loads: ") + e.what());
+ return;
+ }
+ const DraftConfig& c = m.cfg;
+ check(c.generation == 2, "DFlash 2: the generation is read from the architecture");
+ check(m.tensors == 3 + 3 + 15 * c.layers, "DFlash 2: every tensor is accounted for (fc, 2 norms, 3 selector, 15 per layer)");
+ const auto finite = [](const std::vector& v) {
+ for (float f : v)
+ if (!std::isfinite(f)) return false;
+ return !v.empty();
+ };
+ check(finite(decode_projection(m.layers[0].q)), "DFlash 2: a Q4_K projection decodes finite");
+ check(finite(decode_plain(m.layers[0].attn_conv_base)), "DFlash 2: a convolution base kernel decodes finite");
+ std::vector row(size_t(c.selector_rank));
+ decode_row(m.selector_succ, int(c.mask_token_id), row.data());
+ check(finite(row), "DFlash 2: a codebook row decodes finite");
+ bool refused = false;
+ try {
+ decode_row(m.selector_succ, c.vocab, row.data());
+ } catch (const std::runtime_error&) {
+ refused = true;
+ }
+ check(refused, "DFlash 2: a codebook row past the vocabulary is refused");
+ std::printf("DFlash 2 package: %d tensors, %d layers, hidden %d, block %d, taps", m.tensors, c.layers, c.hidden, c.block_size);
+ for (int l : c.target_layer_ids) std::printf(" %d", l);
+ std::printf(", conv %d x groups of %d, selector rank %d top %d, vocab %d\n", c.conv_kernel, c.conv_group, c.selector_rank,
+ c.selector_top_k, c.vocab);
+}
+
int main(int argc, char** argv) {
const fs::path root = fs::temp_directory_path() / ("dflash-draft-test-" + std::to_string(::getpid()));
fs::remove_all(root);
@@ -431,6 +470,9 @@ int main(int argc, char** argv) {
const char* real = (argc > 1 && argv[1][0] != '\0') ? argv[1] : std::getenv("ONEBIT_TEST_DFLASH_DRAFT");
if (real != nullptr && *real != '\0') test_real(real);
else std::printf("SKIP the real package: pass its directory, or set ONEBIT_TEST_DFLASH_DRAFT\n");
+ const char* real2 = (argc > 2 && argv[2][0] != '\0') ? argv[2] : std::getenv("ONEBIT_TEST_DFLASH2_DRAFT");
+ if (real2 != nullptr && *real2 != '\0') test_real2(real2);
+ else std::printf("SKIP the DFlash 2 package: pass its directory second, or set ONEBIT_TEST_DFLASH2_DRAFT\n");
if (g_failures == 0) std::printf("dflash_draft: ok (%d checks)\n", g_checks);
return g_failures == 0 ? 0 : 1;
}
diff --git a/tests/dflash_forward_test.cpp b/tests/dflash_forward_test.cpp
index 938b632cd..0ceb3a442 100644
--- a/tests/dflash_forward_test.cpp
+++ b/tests/dflash_forward_test.cpp
@@ -269,6 +269,27 @@ void properties() {
}
}
+// The context cache: grown in steps it gives exactly the pass draft_forward() gives, and a pass can follow every step.
+void cache_steps() {
+ const DraftWeights w = random_model(5);
+ Rng r{77};
+ const int ctx = 5, block = 3, H = 8;
+ const std::vector taps = r.vec(size_t(ctx) * 16, 1.0f);
+ const std::vector noise = r.vec(size_t(block) * size_t(H), 1.0f);
+ const std::vector fused = fuse_context(w, taps.data(), ctx);
+ DraftKvCache c;
+ c.pos0 = 3;
+ cache_extend(w, c, fused.data(), 2);
+ const std::vector two = draft_forward_cached(w, c, noise.data(), block);
+ check(max_diff(two, draft_forward(w, taps.data(), 2, noise.data(), block, 3)) == 0.0,
+ "cache: two rows, then a pass, is the pass over those two rows");
+ cache_extend(w, c, fused.data() + 2 * H, 1);
+ cache_extend(w, c, fused.data() + 3 * H, 2);
+ check(c.rows == ctx, "cache: rows add up");
+ check(max_diff(draft_forward_cached(w, c, noise.data(), block), draft_forward(w, taps.data(), ctx, noise.data(), block, 3)) == 0.0,
+ "cache: grown 2 + 1 + 2 rows, it gives exactly draft_forward's pass over all 5");
+}
+
void refusals() {
const DraftWeights w = random_model(3);
const std::vector noise(8, 0.5f);
@@ -327,10 +348,40 @@ void real_package(const std::string& dir) {
}
}
+// DFlash 2's two extra pieces, each against arithmetic written out here.
+void dflash2_pieces() {
+ { // 3 rows of 4, groups of 2, a kernel of 2 offsets. base[offset][i], dyn[row][offset][group].
+ const float x[12] = {1, 2, 3, 4, 5, 6, 7, 8, -1, 0, 2, -2};
+ const float base[8] = {1, 1, 1, 1, 0.5f, 0, -1, 2}; // offset 0: identity; offset 1: per element
+ const float dyn[12] = {0, 0, 9, 9, 1, -1, 0.5f, 0, 0, 2, -1, 1};
+ // row 0: only offset 0 exists; (1 + 0) * x0 (its offset-1 kernel, the 9s, has nothing to read)
+ // row 1: (1 + 1) * 5, (1 + 1) * 6, (1 - 1) * 7, (1 - 1) * 8 + (0.5 + 0.5) * 1, (0 + 0.5) * 2, (-1 + 0) * 3, (2 + 0) * 4
+ // row 2: (1 + 0) * -1, (1 + 0) * 0, (1 + 2) * 2, (1 + 2) * -2 + (0.5 - 1) * 5, (0 - 1) * 6, (-1 + 1) * 7, (2 + 1) * 8
+ const float want[12] = {1, 2, 3, 4, 10 + 1, 12 + 1, 0 - 3, 0 + 8, -1 - 2.5f, 0 - 6, 6 + 0, -6 + 24};
+ const std::vector got = draft_conv(x, 3, 4, 2, 2, base, dyn);
+ bool same = got.size() == 12;
+ for (int i = 0; i < 12 && same; i++) same = std::fabs(got[size_t(i)] - want[i]) < 1e-6;
+ check(same, "DFlash 2 convolution: three rows against the hand-computed ones (a row's own kernel, the row before, "
+ "the group a value belongs to, nothing before the first row)");
+ }
+ { // 3 candidates, rank 2. scores: 1.0 + (2*0.5*1 + -1*4*0) = 2.0; 1.5 + (2*0.5*-1 + -1*4*0.5) = -1.5; 0.2 + (2*0.5*2 + -1*4*-0.25) = 3.2
+ const float unary[3] = {1.0f, 1.5f, 0.2f}, prev[2] = {2.0f, -1.0f}, h[2] = {0.5f, 4.0f};
+ const float succ[6] = {1.0f, 0.0f, -1.0f, 0.5f, 2.0f, -0.25f};
+ check(selector_pick(unary, 3, prev, h, succ, 2) == 2,
+ "selector: the pairwise term can lift the lowest-logit candidate over the highest");
+ const float flat[6] = {0, 0, 0, 0, 0, 0};
+ check(selector_pick(unary, 3, prev, h, flat, 2) == 1, "selector: with no pairwise term it is the plain argmax");
+ const float tie[3] = {1.0f, 1.0f, 1.0f};
+ check(selector_pick(tie, 3, prev, h, flat, 2) == 0, "selector: the first of equals is kept");
+ }
+}
+
} // namespace
int main(int argc, char** argv) {
hand_cases();
+ dflash2_pieces();
+ cache_steps();
properties();
refusals();
const char* real = (argc > 1 && argv[1][0] != '\0') ? argv[1] : std::getenv("ONEBIT_TEST_DFLASH_DRAFT");