diff --git a/docs/amd-drafters.md b/docs/amd-drafters.md index 216c5a1f5..be2aca7c8 100644 --- a/docs/amd-drafters.md +++ b/docs/amd-drafters.md @@ -166,6 +166,29 @@ next step of engine#1780. What `dflash_forward_test` establishes: - with a package directory (`-DONEBIT_DFLASH_DRAFT=` or `ONEBIT_TEST_DFLASH_DRAFT`), the real draft decodes and one pass over a made-up context is finite and position-independent. +### DFlash 2 packages (engine#1781) + +`load_draft()` also reads `DFlash2DraftModel` packages (`FastFlowLM/Qwen3.8-27B-DFlash2-NPU2` is the one on this box), +as generation 2 of the same loader. The config states the draft's interface under `dflash_config` and the rotation's base +under `rope_parameters`. Every projection, the fusion matrix included, is a grid of 4736-byte Q4_K chunks rather than +5120-byte q4_1 tiles. Each layer also carries two grouped convolutions, around the attention and around the MLP, and the +package carries a candidate selector (a projection of the hidden state and two vocabulary-sized codebooks). +`decode_draft()` fills the convolutions; the codebooks are read a row at a time with `decode_row()`, since each is +254 MB as f32. + +The forward gains what generation 2 needs, each piece held to arithmetic written out in `dflash_forward_test`: + +| piece | what it computes | +|---|---| +| `draft_conv()` | one half of the grouped convolution: a row's output sums, over the kernel's offsets, (base kernel + the row's own kernel for its group) times the row that many places earlier | +| `selector_pick()` | the selector's choice at one block row: among the target head's top candidates, the highest of logit + the pairwise term between the previous pick and the candidate | +| `DraftKvCache`, `cache_extend()`, `draft_forward_cached()` | the context's keys and values per layer, computed once per context row and reused by every later pass; grown in steps it gives exactly the single-call pass | +| `set_gemm_hook()` | an optional place for the draft's projections to run (the device route); unset, everything runs on the host | + +Measured with these pieces against the served 27B, in a local loop: on a code prompt, 6.86 tokens per round and 7.6-7.8 +tok/s against 4.2 tok/s plain decoding; output identical to plain greedy decoding. The loop is not part of this change; +the numbers are on engine#1781. + ## PARD-2's repos are weights-only — borrow the target's tokenizer `amd/PARD2-*` ship **`README.md`, `config.json`, `model.safetensors`, `warp_model.bin`** and nothing else: **no tokenizer diff --git a/npu/dflash_draft.cpp b/npu/dflash_draft.cpp index 85a0cec8b..3c60bf7f8 100644 --- a/npu/dflash_draft.cpp +++ b/npu/dflash_draft.cpp @@ -65,6 +65,73 @@ float need_float(const json& c, const char* key, const std::string& path) { bool close_to(double a, double b) { return std::fabs(a - b) <= 1e-5 * std::max(1.0, std::fabs(b)); } +void read_target_layers(const json& from, const std::string& path, DraftConfig& o) { + if (!from.contains("target_layer_ids") || !from["target_layer_ids"].is_array() || from["target_layer_ids"].empty()) + fail(path + ": no 'target_layer_ids'"); + int prev = -1; + for (const auto& v : from["target_layer_ids"]) { + if (!v.is_number_integer() || v.get() < 0 || v.get() > 4096) + fail(path + ": 'target_layer_ids' holds something that is not a layer index"); + const int id = int(v.get()); + if (id <= prev) fail(path + ": 'target_layer_ids' is not strictly increasing at " + std::to_string(id)); + o.target_layer_ids.push_back(id); + prev = id; + } +} + +// DFlash2DraftModel: the geometry is at the top level as in any Qwen3 config, the draft's interface is under +// "dflash_config", the rotation's base under "rope_parameters", and the weight file has the family's fixed name. +DraftConfig read_generation2(const json& c, const std::string& path) { + DraftConfig o; + o.generation = 2; + o.hidden = need_int(c, "hidden_size", path); + o.intermediate = need_int(c, "intermediate_size", path); + o.layers = need_int(c, "num_hidden_layers", path); + o.heads = need_int(c, "num_attention_heads", path); + o.kv_heads = need_int(c, "num_key_value_heads", path); + o.head_dim = need_int(c, "head_dim", path); + o.vocab = need_int(c, "vocab_size", path); + o.sliding_window = need_int(c, "sliding_window", path); + o.rms_eps = need_float(c, "rms_norm_eps", path); + if (!c.contains("rope_parameters") || !c["rope_parameters"].is_object()) + fail(path + ": no 'rope_parameters'"); + o.rope_theta = need_float(c["rope_parameters"], "rope_theta", path + " rope_parameters"); + o.embedding_scale = 1.0f; + if (!c.contains("dflash_config") || !c["dflash_config"].is_object()) fail(path + ": no 'dflash_config'"); + const json& d = c["dflash_config"]; + const std::string dp = path + " dflash_config"; + o.block_size = need_int(d, "block_size", dp); + o.conv_kernel = need_int(d, "conv_kernel_size", dp); + o.conv_group = need_int(d, "conv_group_size", dp); + o.selector_rank = need_int(d, "selector_rank", dp); + o.selector_top_k = need_int(d, "selector_top_k", dp); + if (!d.contains("mask_token_id") || !d["mask_token_id"].is_number_integer() || d["mask_token_id"].get() < 0) + fail(dp + ": no usable 'mask_token_id'"); + o.mask_token_id = d["mask_token_id"].get(); + read_target_layers(d, dp, o); + if (c.contains("is_causal")) { + if (!c["is_causal"].is_boolean()) fail(path + ": 'is_causal' is not a boolean"); + o.causal = c["is_causal"].get(); + } + o.draft_weight = "dflash_weight.q4nx"; + if (c.contains("draft_weight")) { + if (!c["draft_weight"].is_string() || c["draft_weight"].get().empty()) fail(path + ": 'draft_weight' is not a file name"); + o.draft_weight = c["draft_weight"].get(); + if (o.draft_weight.find('/') != std::string::npos || o.draft_weight.find("..") != std::string::npos) + fail(path + ": 'draft_weight' must be a file name inside the package, got '" + o.draft_weight + "'"); + } + if (o.kv_heads > o.heads || o.heads % o.kv_heads != 0) + fail(path + ": " + std::to_string(o.heads) + " heads cannot share " + std::to_string(o.kv_heads) + " key/value heads"); + if (o.block_size < 2) fail(path + ": block_size " + std::to_string(o.block_size) + " drafts nothing"); + if (o.conv_kernel < 1 || o.conv_group < 1 || o.hidden % o.conv_group != 0) + fail(dp + ": a convolution of kernel " + std::to_string(o.conv_kernel) + " over groups of " + + std::to_string(o.conv_group) + " does not fit hidden_size " + std::to_string(o.hidden)); + if (o.selector_rank < 1 || o.selector_top_k < 1 || o.selector_top_k > o.vocab) + fail(dp + ": selector_rank or selector_top_k is not usable"); + if (o.causal) fail(path + ": a causal draft is not read by this loader"); + return o; +} + } // namespace DraftConfig read_draft_config(const std::string& package_dir) { @@ -79,15 +146,18 @@ DraftConfig read_draft_config(const std::string& package_dir) { } if (!c.is_object()) fail(path + ": not a JSON object"); const std::string type = c.value("model_type", json("")).is_string() ? c.value("model_type", std::string()) : ""; - if (type != "dflash") fail(path + ": model_type is '" + type + "', not 'dflash'"); - // The architecture decides which tensors exist. This loader knows one; a package naming another is a different - // model (DFlash2 adds a selector and a convolution), and reading it with these rules would drop those silently. + // The architecture decides which tensors exist, so it is read first. Two are known; a package naming another is a + // different model, and reading it with these rules would drop what it adds silently. + std::string arch; if (c.contains("architectures")) { if (!c["architectures"].is_array() || c["architectures"].empty() || !c["architectures"][0].is_string()) fail(path + ": 'architectures' is not a list of names"); - const std::string arch = c["architectures"][0].get(); - if (arch != "DFlashDraftModel") fail(path + ": architecture '" + arch + "' is not read by this loader (DFlashDraftModel is)"); + arch = c["architectures"][0].get(); } + if (arch == "DFlash2DraftModel") return read_generation2(c, path); + if (type != "dflash") fail(path + ": model_type is '" + type + "', not 'dflash'"); + if (!arch.empty() && arch != "DFlashDraftModel") + fail(path + ": architecture '" + arch + "' is not read by this loader (DFlashDraftModel and DFlash2DraftModel are)"); DraftConfig o; o.hidden = need_int(c, "hidden_size", path); o.intermediate = need_int(c, "intermediate_size", path); @@ -103,17 +173,7 @@ DraftConfig read_draft_config(const std::string& package_dir) { if (!c.contains("mask_token_id") || !c["mask_token_id"].is_number_integer() || c["mask_token_id"].get() < 0) fail(path + ": no usable 'mask_token_id'"); o.mask_token_id = c["mask_token_id"].get(); - if (!c.contains("target_layer_ids") || !c["target_layer_ids"].is_array() || c["target_layer_ids"].empty()) - fail(path + ": no 'target_layer_ids'"); - int prev = -1; - for (const auto& v : c["target_layer_ids"]) { - if (!v.is_number_integer() || v.get() < 0 || v.get() > 4096) - fail(path + ": 'target_layer_ids' holds something that is not a layer index"); - const int id = int(v.get()); - if (id <= prev) fail(path + ": 'target_layer_ids' is not strictly increasing at " + std::to_string(id)); - o.target_layer_ids.push_back(id); - prev = id; - } + read_target_layers(c, path, o); if (!c.contains("draft_weight") || !c["draft_weight"].is_string() || c["draft_weight"].get().empty()) fail(path + ": no 'draft_weight'"); o.draft_weight = c["draft_weight"].get(); @@ -172,13 +232,20 @@ DraftModel load_draft(const std::string& package_dir) { v.cols = cols; return v; }; + // A generation-2 package stores its projections as Q4_K chunks (4736 B per 32 x 256), not q4_1 tiles (5120 B). + const int chunk = c.generation == 2 ? Q4NX_CHUNK_Q4K : Q4NX_TILE_BYTES; const auto projection = [&](const std::string& name, int n, int k) { if (n % 32 != 0 || k % 256 != 0) fail(path + ": '" + name + "' would be " + std::to_string(n) + " x " + std::to_string(k) + ", which is not a whole 32 x 256 tile grid"); - return take(name, "I8", q4nx_tensor_bytes(n, k), n, k); + TensorView v = take(name, "I8", size_t(n / 32) * size_t(k / 256) * size_t(chunk), n, k); + v.chunk = chunk; + return v; }; const auto gain = [&](const std::string& name, int width) { return take(name, "BF16", size_t(width) * 2, 1, width); }; + const auto plain = [&](const std::string& name, int rows, int cols) { + return take(name, "BF16", size_t(rows) * size_t(cols) * 2, rows, cols); + }; // The container repeats the interface scalars; a package whose two halves disagree is not one model. const auto scalar = [&](const std::string& name, const char* dtype, size_t count) { @@ -189,7 +256,7 @@ DraftModel load_draft(const std::string& package_dir) { const auto agree = [&](bool same, const std::string& name, const std::string& got, const std::string& want) { if (!same) fail(path + ": '" + name + "' is " + got + " and config.json says " + want); }; - { + if (c.generation == 1) { const uint8_t* p = scalar("dflash.block_size", "I32", 1); agree(i32_at(p, 0) == c.block_size, "dflash.block_size", std::to_string(i32_at(p, 0)), std::to_string(c.block_size)); p = scalar("dflash.sliding_window", "I32", 1); @@ -210,8 +277,15 @@ DraftModel load_draft(const std::string& package_dir) { // and the declared shape must be that blocking -- a plain [hidden, taps*hidden] label would mean another element order. if (c.hidden % 32 != 0) fail(path + ": hidden_size " + std::to_string(c.hidden) + " does not fill 32-row blocks"); const int fused = c.taps() * c.hidden; - m.fc = take("model.fc.weight", "BF16", size_t(c.hidden) * size_t(fused) * 2, c.hidden, fused); - { + if (c.generation == 2) { + m.fc = projection("model.fc.weight", c.hidden, fused); + const int groups = c.hidden / c.conv_group; + m.selector_hproj = plain("model.candidate_selector.hidden_projection.weight", c.selector_rank, c.hidden); + m.selector_pred = plain("model.candidate_selector.predecessor_codebook", c.vocab, c.selector_rank); + m.selector_succ = plain("model.candidate_selector.successor_codebook", c.vocab, c.selector_rank); + (void)groups; + } else { + m.fc = take("model.fc.weight", "BF16", size_t(c.hidden) * size_t(fused) * 2, c.hidden, fused); const std::vector want = {c.hidden / 32, fused, 32}; if (by_name["model.fc.weight"]->shape != want) fail(path + ": 'model.fc.weight' is not declared [hidden/32, taps*hidden, 32], so its element order is not the one this loader undoes"); @@ -232,11 +306,18 @@ DraftModel load_draft(const std::string& package_dir) { L.post_norm = gain(p + "post_attention_layernorm.weight", c.hidden); L.q_norm = gain(p + "self_attn.q_norm.weight", c.head_dim); L.k_norm = gain(p + "self_attn.k_norm.weight", c.head_dim); + if (c.generation == 2) { + const int kernels = 2 * c.conv_kernel, groups = c.hidden / c.conv_group; + L.attn_conv_base = plain(p + "attention_conv.base_kernel", kernels, c.hidden); + L.attn_conv_proj = plain(p + "attention_conv.kernel_projection.weight", kernels * groups, c.hidden); + L.mlp_conv_base = plain(p + "mlp_conv.base_kernel", kernels, c.hidden); + L.mlp_conv_proj = plain(p + "mlp_conv.kernel_projection.weight", kernels * groups, c.hidden); + } m.layers.push_back(L); } for (const auto& t : table.tensors) if (used.count(t.name) == 0) - fail(path + ": tensor '" + t.name + "' is not part of a DFlashDraftModel (" + std::to_string(table.tensors.size() - used.size()) + + fail(path + ": tensor '" + t.name + "' is not part of this draft's architecture (" + std::to_string(table.tensors.size() - used.size()) + " unexplained): this is another architecture"); return m; } @@ -246,9 +327,11 @@ std::vector decode_projection(const TensorView& t) { std::vector out(size_t(t.rows) * size_t(t.cols)); // Not the size-dispatching q4nx_dequant_tensor(): a projection here is a tile grid by contract, and a grid whose // byte count happened to equal rows*cols*2 must not be read as plain 2-byte floats. - if (t.bytes != q4nx_tensor_bytes(t.rows, t.cols) || - q4nx_dequant_tensor_chunked(t.data, t.bytes, t.rows, t.cols, 5120, out.data()) != 0) - fail("'" + t.name + "' does not decode as a " + std::to_string(t.rows) + " x " + std::to_string(t.cols) + " q4_1 tile grid"); + const int chunk = t.chunk > 0 ? t.chunk : Q4NX_TILE_BYTES; + if (t.rows % 32 != 0 || t.cols % 256 != 0 || t.bytes != size_t(t.rows / 32) * size_t(t.cols / 256) * size_t(chunk) || + q4nx_dequant_tensor_chunked(t.data, t.bytes, t.rows, t.cols, chunk, out.data()) != 0) + fail("'" + t.name + "' does not decode as a " + std::to_string(t.rows) + " x " + std::to_string(t.cols) + " grid of " + + std::to_string(chunk) + "-byte tiles"); return out; } @@ -259,6 +342,20 @@ std::vector decode_gain(const TensorView& t) { return out; } +std::vector decode_plain(const TensorView& t) { + if (!t.ok() || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) fail("decode_plain of '" + t.name + "': not 2 bytes per element"); + std::vector out(size_t(t.rows) * size_t(t.cols)); + for (size_t i = 0; i < out.size(); ++i) out[i] = bf16(t.data + i * 2); + return out; +} + +void decode_row(const TensorView& t, int row, float* out) { + if (!t.ok() || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) fail("decode_row of '" + t.name + "': not 2 bytes per element"); + if (row < 0 || row >= t.rows || out == nullptr) fail("decode_row of '" + t.name + "': row " + std::to_string(row) + " is outside the tensor"); + const uint8_t* p = t.data + size_t(row) * size_t(t.cols) * 2; + for (int i = 0; i < t.cols; ++i) out[i] = bf16(p + size_t(i) * 2); +} + std::vector decode_fusion(const TensorView& t) { if (!t.ok() || t.rows % 32 != 0 || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) fail("decode_fusion of '" + t.name + "': not hidden x fused at 2 bytes per element"); diff --git a/npu/dflash_draft.h b/npu/dflash_draft.h index d5e72ad51..42831732b 100644 --- a/npu/dflash_draft.h +++ b/npu/dflash_draft.h @@ -59,6 +59,13 @@ struct DraftConfig { float rope_theta = 0.0f; float embedding_scale = 0.0f; bool causal = false; + // Which draft this is. 1: DFlashDraftModel (model_type "dflash"). 2: DFlash2DraftModel, which adds a grouped + // convolution around each layer's attention and MLP and a candidate selector, states its interface under + // "dflash_config", and stores every projection (the fusion matrix included) as Q4_K chunks. + int generation = 1; + int conv_kernel = 0, conv_group = 0; // generation 2 + int selector_rank = 0, selector_top_k = 0; // generation 2 + int vocab = 0; // generation 2: rows of the selector's codebooks std::string draft_weight; // the container's file name, relative to the package std::string base_model; // the target this draft was made for, as the package names it (may be empty) @@ -74,17 +81,25 @@ struct TensorView { size_t bytes = 0; int rows = 0; // N (out); 1 for a vector int cols = 0; // K (in), or the vector's length + int chunk = 0; // a projection's bytes per 32 x 256 tile: 0 or 5120 = q4_1, 4736 = Q4_K bool ok() const { return data != nullptr && bytes != 0; } }; struct DraftLayer { TensorView q, k, v, o, gate, up, down; // q4_1 tile grids, N x K TensorView input_norm, post_norm, q_norm, k_norm; // 2-byte gains + // Generation 2, 2 bytes per element: base kernels [2][kernel][hidden], kernel projections + // [2 * kernel * hidden / group][hidden] row-major. + TensorView attn_conv_base, attn_conv_proj, mlp_conv_base, mlp_conv_proj; }; struct DraftModel { DraftConfig cfg; - TensorView fc; // hidden x (taps * hidden): fuses the target layers' hidden states + TensorView fc; // hidden x (taps * hidden): fuses the target layers' hidden states (generation 1: 2-byte + // elements in 32-row blocks, decode_fusion(); generation 2: a tile grid, decode_projection()) + // Generation 2: the candidate selector. hidden_projection [rank][hidden], the two codebooks [vocab][rank], all at + // 2 bytes per element. The codebooks are 254 MB each as f32, so a caller reads rows out of the view (decode_row()). + TensorView selector_hproj, selector_pred, selector_succ; TensorView hidden_norm; // applied to the fused features TensorView norm; // the final norm std::vector layers; @@ -105,6 +120,12 @@ std::vector decode_projection(const TensorView& t); // A norm's gains as f32. std::vector decode_gain(const TensorView& t); +// A plain 2-byte-per-element tensor as f32, row-major [rows][cols]. +std::vector decode_plain(const TensorView& t); + +// One row of a plain 2-byte-per-element tensor into out[cols]. Throws if the row is outside the tensor. +void decode_row(const TensorView& t, int row, float* out); + // The fusion matrix as f32, row-major [hidden][taps * hidden]. The container stores it as [hidden/32][taps*hidden][32] // -- 32-row blocks, column by column, the row inside the block last -- and that is the order this undoes: // out[(a*32 + r) * cols + k] = stored[(a * cols + k) * 32 + r]. diff --git a/npu/dflash_forward.cpp b/npu/dflash_forward.cpp index a9c56a9e6..5d730498b 100644 --- a/npu/dflash_forward.cpp +++ b/npu/dflash_forward.cpp @@ -31,14 +31,22 @@ void need(bool ok, const std::string& what) { if (!ok) throw std::runtime_error("dflash forward: " + what); } +GemmHook g_hook = nullptr; +void* g_hook_user = nullptr; + // y[rows][n_out] = x[rows][n_in] * W^T, W row-major [n_out][n_in]. std::vector linear(const std::vector& W, int n_out, int n_in, const float* x, int rows) { std::vector y(size_t(rows) * size_t(n_out)); - for (int r = 0; r < rows; r++) { - const float* xr = x + size_t(r) * size_t(n_in); - for (int n = 0; n < n_out; n++) { - const float* wr = W.data() + size_t(n) * size_t(n_in); + if (g_hook && rows > 0 && g_hook(W.data(), n_out, n_in, x, rows, y.data(), g_hook_user)) return y; + // Output rows are independent, so they are shared out across threads when the build has OpenMP; the sum itself + // stays in double and may be taken in vector lanes, so two builds can differ in the last bits of a result. + #pragma omp parallel for schedule(static) + for (int n = 0; n < n_out; n++) { + const float* wr = W.data() + size_t(n) * size_t(n_in); + for (int r = 0; r < rows; r++) { + const float* xr = x + size_t(r) * size_t(n_in); double acc = 0.0; + #pragma omp simd reduction(+ : acc) for (int k = 0; k < n_in; k++) acc += double(wr[k]) * double(xr[k]); y[size_t(r) * size_t(n_out) + size_t(n)] = float(acc); } @@ -69,8 +77,76 @@ void rope(float* v, int d, long pos, float theta) { } } +// One half of a DFlash 2 convolution over `rows` rows of H. `dyn` is [rows][kernel][groups] for this half. +std::vector convolve(const float* x, int rows, int H, int kernel, int group, const float* base, const float* dyn) { + const int groups = H / group; + std::vector out(size_t(rows) * size_t(H), 0.0f); + for (int t = 0; t < rows; t++) + for (int off = 0; off < kernel && off <= t; off++) { + const float* src = x + size_t(t - off) * size_t(H); + const float* b = base + size_t(off) * size_t(H); + const float* d = dyn + (size_t(t) * size_t(kernel) + size_t(off)) * size_t(groups); + float* o = out.data() + size_t(t) * size_t(H); + for (int g = 0; g < groups; g++) + for (int j = 0; j < group; j++) { + const int i = g * group + j; + o[i] += (b[i] + d[g]) * src[i]; + } + } + return out; +} + +// prepare(): the row's kernels from the row, the first half applied now, the second half kept for finish(). +std::vector conv_prepare(const DraftWeights& w, const DraftConv& c, const std::vector& x, int rows, + std::vector& keep) { + const int H = w.hidden, K = w.conv_kernel, G = H / w.conv_group; + const std::vector dyn = linear(c.proj, 2 * K * G, H, x.data(), rows); // [rows][2][K][G] + std::vector first(size_t(rows) * size_t(K) * size_t(G)); + keep.assign(first.size(), 0.0f); + for (int t = 0; t < rows; t++) { + const float* d = dyn.data() + size_t(t) * size_t(2 * K * G); + std::copy(d, d + K * G, first.begin() + ptrdiff_t(size_t(t) * size_t(K * G))); + std::copy(d + K * G, d + 2 * K * G, keep.begin() + ptrdiff_t(size_t(t) * size_t(K * G))); + } + return convolve(x.data(), rows, H, K, w.conv_group, c.base.data(), first.data()); +} + +std::vector conv_finish(const DraftWeights& w, const DraftConv& c, const std::vector& x, int rows, + const std::vector& keep) { + return convolve(x.data(), rows, w.hidden, w.conv_kernel, w.conv_group, + c.base.data() + size_t(w.conv_kernel) * size_t(w.hidden), keep.data()); +} + } // namespace +std::vector draft_conv(const float* x, int rows, int hidden, int kernel, int group, const float* base, + const float* dyn) { + need(x && base && dyn && rows > 0 && hidden > 0 && kernel > 0 && group > 0 && hidden % group == 0, + "a convolution with an empty input, or a group size that does not divide the hidden width"); + return convolve(x, rows, hidden, kernel, group, base, dyn); +} + +int selector_pick(const float* unary, int k, const float* prev, const float* h, const float* succ, int rank) { + need(unary && prev && h && succ && k > 0 && rank > 0, "a selector row with no candidates or no codebook rows"); + int best = 0; + double top = 0.0; + for (int c = 0; c < k; c++) { + double score = unary[c]; + const float* s = succ + size_t(c) * size_t(rank); + for (int r = 0; r < rank; r++) score += double(prev[r]) * double(h[r]) * double(s[r]); + if (c == 0 || score > top) { + top = score; + best = c; + } + } + return best; +} + +void set_gemm_hook(GemmHook hook, void* user) { + g_hook = hook; + g_hook_user = user; +} + void check_weights(const DraftWeights& w) { need(w.hidden > 0 && w.intermediate > 0 && w.heads > 0 && w.kv_heads > 0 && w.head_dim > 0 && w.taps > 0, "a geometry field is zero or negative"); @@ -94,6 +170,18 @@ void check_weights(const DraftWeights& w) { need(L.input_norm.size() == H && L.post_norm.size() == H, at + "a layer norm is not hidden wide"); need(L.q_norm.size() == size_t(w.head_dim) && L.k_norm.size() == size_t(w.head_dim), at + "q_norm or k_norm is not head_dim wide"); + for (const DraftConv* c : {&L.attn_conv, &L.mlp_conv}) { + if (w.conv_kernel == 0) { + need(!c->present(), at + "a convolution on a draft that declares none"); + continue; + } + need(w.conv_kernel > 0 && w.conv_group > 0 && w.hidden % w.conv_group == 0, + "the convolution's kernel or group size does not fit the hidden width"); + const size_t G = size_t(w.hidden / w.conv_group); + need(c->base.size() == 2 * size_t(w.conv_kernel) * H, at + "a convolution's base kernel is not 2 x kernel x hidden"); + need(c->proj.size() == 2 * size_t(w.conv_kernel) * G * H, + at + "a convolution's kernel projection is not (2 * kernel * groups) x hidden"); + } } } @@ -107,7 +195,9 @@ DraftWeights decode_draft(const DraftModel& m) { w.taps = m.cfg.taps(); w.rms_eps = m.cfg.rms_eps; w.rope_theta = m.cfg.rope_theta; - w.fc = decode_fusion(m.fc); + w.fc = m.cfg.generation == 2 ? decode_projection(m.fc) : decode_fusion(m.fc); + w.conv_kernel = m.cfg.conv_kernel; + w.conv_group = m.cfg.conv_group; w.hidden_norm = decode_gain(m.hidden_norm); w.norm = decode_gain(m.norm); w.layers.reserve(m.layers.size()); @@ -124,53 +214,113 @@ DraftWeights decode_draft(const DraftModel& m) { d.post_norm = decode_gain(L.post_norm); d.q_norm = decode_gain(L.q_norm); d.k_norm = decode_gain(L.k_norm); + if (m.cfg.generation == 2) { + d.attn_conv.base = decode_plain(L.attn_conv_base); + d.attn_conv.proj = decode_plain(L.attn_conv_proj); + d.mlp_conv.base = decode_plain(L.mlp_conv_base); + d.mlp_conv.proj = decode_plain(L.mlp_conv_proj); + } w.layers.push_back(std::move(d)); } check_weights(w); return w; } +std::vector fuse_context(const DraftWeights& w, const float* taps, int ctx) { + check_weights(w); + need(ctx >= 0 && (ctx == 0 || taps != nullptr), "a context length without context rows"); + std::vector c; + if (ctx > 0) { + c = linear(w.fc, w.hidden, w.hidden * w.taps, taps, ctx); + rms_norm(c.data(), size_t(ctx), w.hidden, w.hidden_norm, w.rms_eps); + } + return c; +} + std::vector draft_forward(const DraftWeights& w, const float* taps, int ctx, const float* noise, int block, long pos0, int window) { + const std::vector c = fuse_context(w, taps, ctx); + return draft_forward_fused(w, c.data(), ctx, noise, block, pos0, window); +} + +void cache_extend(const DraftWeights& w, DraftKvCache& c, const float* fused_rows, int n) { + check_weights(w); + need(n >= 0 && (n == 0 || fused_rows != nullptr), "context rows to add without the rows"); + const int H = w.hidden, D = w.head_dim, KV = w.kv_heads * D; + if (c.k.size() != w.layers.size()) { + need(c.rows == 0, "a cache whose layer count does not match the draft"); + c.k.assign(w.layers.size(), {}); + c.v.assign(w.layers.size(), {}); + } + if (n == 0) return; + for (size_t l = 0; l < w.layers.size(); l++) { + const DraftLayerWeights& L = w.layers[l]; + std::vector k = linear(L.k, KV, H, fused_rows, n); + const std::vector v = linear(L.v, KV, H, fused_rows, n); + rms_norm(k.data(), size_t(n) * size_t(w.kv_heads), D, L.k_norm, w.rms_eps); + for (int t = 0; t < n; t++) + for (int hd = 0; hd < w.kv_heads; hd++) + rope(k.data() + size_t(t) * size_t(KV) + size_t(hd) * size_t(D), D, c.pos0 + c.rows + t, w.rope_theta); + c.k[l].insert(c.k[l].end(), k.begin(), k.end()); + c.v[l].insert(c.v[l].end(), v.begin(), v.end()); + } + c.rows += n; +} + +std::vector draft_forward_fused(const DraftWeights& w, const float* fused, int ctx, const float* noise, int block, + long pos0, int window) { + need(ctx >= 0 && (ctx == 0 || fused != nullptr), "a context length without context rows"); + DraftKvCache c; + c.pos0 = pos0; + cache_extend(w, c, fused, ctx); + return draft_forward_cached(w, c, noise, block, window); +} + +std::vector draft_forward_cached(const DraftWeights& w, const DraftKvCache& c, const float* noise, int block, + int window) { check_weights(w); need(block > 0 && noise != nullptr, "an empty block"); - need(ctx >= 0 && (ctx == 0 || taps != nullptr), "a context length without context rows"); need(window >= 0, "a negative window"); + need(c.rows == 0 || c.k.size() == w.layers.size(), "a cache that was not built for this draft"); const int H = w.hidden, I = w.intermediate, D = w.head_dim; const int Q = w.heads * D, KV = w.kv_heads * D; + const int ctx = c.rows; + const long pos0 = c.pos0; const int keys = ctx + block; const int group = w.heads / w.kv_heads; // query heads per key/value head, consecutive - // The context: fused, normalised once, and the same rows for every layer. - std::vector c; - if (ctx > 0) { - c = linear(w.fc, H, H * w.taps, taps, ctx); - rms_norm(c.data(), size_t(ctx), H, w.hidden_norm, w.rms_eps); - } - + // The context's keys and values come from the cache: they depend only on the fused context rows and the layer, + // never on the block, so a caller that drafts many blocks over one growing context computes each row once. std::vector x(noise, noise + size_t(block) * size_t(H)); - std::vector kv_in(size_t(keys) * size_t(H)); - std::copy(c.begin(), c.end(), kv_in.begin()); std::vector score(static_cast(keys)); std::vector attn(size_t(block) * size_t(Q)); - for (const DraftLayerWeights& L : w.layers) { + for (size_t l = 0; l < w.layers.size(); l++) { + const DraftLayerWeights& L = w.layers[l]; // attention std::vector h = x; rms_norm(h.data(), size_t(block), H, L.input_norm, w.rms_eps); - std::copy(h.begin(), h.end(), kv_in.begin() + ptrdiff_t(size_t(ctx) * size_t(H))); + std::vector attn_keep; + if (L.attn_conv.present()) h = conv_prepare(w, L.attn_conv, h, block, attn_keep); std::vector q = linear(L.q, Q, H, h.data(), block); - std::vector k = linear(L.k, KV, H, kv_in.data(), keys); - std::vector v = linear(L.v, KV, H, kv_in.data(), keys); + std::vector kb = linear(L.k, KV, H, h.data(), block); + const std::vector vb = linear(L.v, KV, H, h.data(), block); rms_norm(q.data(), size_t(block) * size_t(w.heads), D, L.q_norm, w.rms_eps); - rms_norm(k.data(), size_t(keys) * size_t(w.kv_heads), D, L.k_norm, w.rms_eps); - for (int i = 0; i < block; i++) + rms_norm(kb.data(), size_t(block) * size_t(w.kv_heads), D, L.k_norm, w.rms_eps); + for (int i = 0; i < block; i++) { for (int hd = 0; hd < w.heads; hd++) rope(q.data() + size_t(i) * size_t(Q) + size_t(hd) * size_t(D), D, pos0 + ctx + i, w.rope_theta); - for (int t = 0; t < keys; t++) for (int hd = 0; hd < w.kv_heads; hd++) - rope(k.data() + size_t(t) * size_t(KV) + size_t(hd) * size_t(D), D, pos0 + t, w.rope_theta); + rope(kb.data() + size_t(i) * size_t(KV) + size_t(hd) * size_t(D), D, pos0 + ctx + i, w.rope_theta); + } + // keys and values: the cached context rows, then the block's own + const auto key_at = [&](int t) { + return t < ctx ? c.k[l].data() + size_t(t) * size_t(KV) : kb.data() + size_t(t - ctx) * size_t(KV); + }; + const auto val_at = [&](int t) { + return t < ctx ? c.v[l].data() + size_t(t) * size_t(KV) : vb.data() + size_t(t - ctx) * size_t(KV); + }; const double scale = 1.0 / std::sqrt(double(D)); for (int i = 0; i < block; i++) { @@ -184,7 +334,7 @@ std::vector draft_forward(const DraftWeights& w, const float* taps, int c score[size_t(t)] = -INFINITY; continue; } - const float* kvp = k.data() + size_t(t) * size_t(KV) + size_t(kh) * size_t(D); + const float* kvp = key_at(t) + size_t(kh) * size_t(D); double s = 0.0; for (int d = 0; d < D; d++) s += double(qv[d]) * double(kvp[d]); score[size_t(t)] = s * scale; @@ -199,18 +349,20 @@ std::vector draft_forward(const DraftWeights& w, const float* taps, int c float* out = attn.data() + size_t(i) * size_t(Q) + size_t(hd) * size_t(D); for (int d = 0; d < D; d++) { double acc = 0.0; - for (int t = 0; t < keys; t++) - acc += score[size_t(t)] * double(v[size_t(t) * size_t(KV) + size_t(kh) * size_t(D) + size_t(d)]); + for (int t = 0; t < keys; t++) acc += score[size_t(t)] * double(val_at(t)[size_t(kh) * size_t(D) + size_t(d)]); out[d] = float(acc / sum); } } } std::vector o = linear(L.o, H, Q, attn.data(), block); + if (L.attn_conv.present()) o = conv_finish(w, L.attn_conv, o, block, attn_keep); for (size_t j = 0; j < x.size(); j++) x[j] += o[j]; // MLP std::vector p = x; rms_norm(p.data(), size_t(block), H, L.post_norm, w.rms_eps); + std::vector mlp_keep; + if (L.mlp_conv.present()) p = conv_prepare(w, L.mlp_conv, p, block, mlp_keep); std::vector g = linear(L.gate, I, H, p.data(), block); std::vector u = linear(L.up, I, H, p.data(), block); for (size_t j = 0; j < g.size(); j++) { @@ -218,6 +370,7 @@ std::vector draft_forward(const DraftWeights& w, const float* taps, int c g[j] = float(a / (1.0 + std::exp(-a)) * double(u[j])); // silu(gate) * up } std::vector dn = linear(L.down, H, I, g.data(), block); + if (L.mlp_conv.present()) dn = conv_finish(w, L.mlp_conv, dn, block, mlp_keep); for (size_t j = 0; j < x.size(); j++) x[j] += dn[j]; } diff --git a/npu/dflash_forward.h b/npu/dflash_forward.h index d739c85c9..39431a27b 100644 --- a/npu/dflash_forward.h +++ b/npu/dflash_forward.h @@ -44,21 +44,40 @@ namespace onebit::npu::dflash { +// DFlash 2 only: a grouped convolution over the block rows, applied before and after the attention and before and +// after the MLP. The hidden vector is cut into groups of `group` values. A row's kernel is a fixed part (`base`) plus +// a part computed from the row itself (`proj` of the row, one value per offset and group), and the output at row t +// sums, over offset 0..kernel-1, (base + dynamic) * x[t - offset]; rows before the block count as zero. `prepare` +// uses the first half of `base` and of the projection's output, `finish` the second half. +struct DraftConv { + std::vector base; // [2][kernel][hidden] + std::vector proj; // row-major [2 * kernel * (hidden / group)][hidden] + bool present() const { return !base.empty(); } +}; + struct DraftLayerWeights { std::vector q, k, v, o, gate, up, down; // row-major [out][in] std::vector input_norm, post_norm, q_norm, k_norm; // gains + DraftConv attn_conv, mlp_conv; // empty on a first-generation draft }; // A draft decoded to f32. For the Bolt draft this is about 1.3 GB; the caller decides how long to keep it. struct DraftWeights { int hidden = 0, intermediate = 0, heads = 0, kv_heads = 0, head_dim = 0, taps = 0; float rms_eps = 0.0f, rope_theta = 0.0f; + int conv_kernel = 0, conv_group = 0; // DFlash 2; 0 when the layers carry no convolution std::vector fc; // [hidden][taps * hidden] std::vector hidden_norm; // [hidden] std::vector norm; // [hidden] std::vector layers; }; +// An optional place for the projections to run. When set, every matrix product of the draft forward is offered to +// it first: W is the row-major [n_out][n_in] matrix the product would use (its address identifies the projection), +// x is [rows][n_in], y is [rows][n_out]. Return false to have the host compute it instead. One hook per process. +using GemmHook = bool (*)(const float* W, int n_out, int n_in, const float* x, int rows, float* y, void* user); +void set_gemm_hook(GemmHook hook, void* user); + // Decode every tensor of a loaded package. Throws std::runtime_error if a decoded tensor has the wrong size. DraftWeights decode_draft(const DraftModel& m); @@ -75,4 +94,38 @@ void check_weights(const DraftWeights& w); std::vector draft_forward(const DraftWeights& w, const float* taps, int ctx, const float* noise, int block, long pos0 = 0, int window = 0); +// One half of a DFlash 2 convolution, on its own so it can be held to a hand-computed case: `rows` rows of +// `hidden`, `base` is [kernel][hidden] and `dyn` is [rows][kernel][hidden / group] (a row's own kernel, per offset +// and group). out[t][i] = sum over offset < kernel, offset <= t of (base[offset][i] + dyn[t][offset][i / group]) * +// x[t - offset][i]. Rows before the first count as zero. +std::vector draft_conv(const float* x, int rows, int hidden, int kernel, int group, const float* base, + const float* dyn); + +// The DFlash 2 candidate selector's choice at one block row. `unary` holds the target head's logits of the `k` +// candidates; `prev` is the predecessor codebook's row of the token picked for the row before (the last accepted +// token for the first row), `h` the selector's projection of this row's hidden state, both `rank` wide; `succ` is +// the successor codebook's rows of the candidates, [k][rank]. A candidate's score is +// unary + sum_r prev[r] * h[r] * succ[r]; the highest wins and the first of equals is kept. Returns its index. +int selector_pick(const float* unary, int k, const float* prev, const float* h, const float* succ, int rank); + +// The context's keys and values per layer, for a caller that drafts many blocks over one growing context: they depend +// only on the fused context rows, so each row is projected, normalised and rotated once (at position pos0 + its +// index) and reused by every later pass. draft_forward_fused() builds one for its own call. +struct DraftKvCache { + long pos0 = 0; + int rows = 0; + std::vector> k, v; // per layer, [rows][kv_heads * head_dim] +}; +// Append `n` fused context rows ([n][hidden], from fuse_context()) to the cache. +void cache_extend(const DraftWeights& w, DraftKvCache& c, const float* fused_rows, int n); +// One pass over the cache's context rows (positions pos0 .. pos0 + rows - 1) and a block placed after them. +std::vector draft_forward_cached(const DraftWeights& w, const DraftKvCache& c, const float* noise, int block, + int window = 0); + +// The same pass in two steps, for a caller that drafts many blocks over one growing context: the context rows are +// fused and normalised once ([ctx][hidden]) and handed to every pass that needs a prefix of them. +std::vector fuse_context(const DraftWeights& w, const float* taps, int ctx); +std::vector draft_forward_fused(const DraftWeights& w, const float* fused, int ctx, const float* noise, int block, + long pos0 = 0, int window = 0); + } // namespace onebit::npu::dflash diff --git a/tests/dflash_draft_test.cpp b/tests/dflash_draft_test.cpp index 52735cd82..62fdc83cb 100644 --- a/tests/dflash_draft_test.cpp +++ b/tests/dflash_draft_test.cpp @@ -298,7 +298,7 @@ void test_synthetic(const fs::path& root) { refused(with("short", [&](std::vector& t) { find(t, "model.layers.0.self_attn.k_proj.weight").bytes.resize(5120); }, cfg), "'model.layers.0.self_attn.k_proj.weight' holds 5120 B", "a projection with too few tiles"); refused(with("extra", [&](std::vector& t) { t.push_back({"model.selector.weight", "BF16", "[4]", std::string(8, '\0')}); }, cfg), - "'model.selector.weight' is not part of a DFlashDraftModel", "a tensor of another architecture"); + "'model.selector.weight' is not part of this draft's architecture", "a tensor of another architecture"); refused(with("dtype", [&](std::vector& t) { find(t, "model.norm.weight").dtype = "F16"; }, cfg), "'model.norm.weight' is declared F16", "a gain under another dtype label"); refused(with("plainfc", [&](std::vector& t) { find(t, "model.fc.weight").shape = "[256,512]"; }, cfg), @@ -308,7 +308,9 @@ void test_synthetic(const fs::path& root) { refused(with("taps", [&](std::vector& t) { const int32_t v = 2; std::memcpy(find(t, "dflash.target_layer_ids").bytes.data() + 4, &v, 4); }, cfg), "'dflash.target_layer_ids[1]' is 2 and config.json says 3", "a container that disagrees on a target layer"); refused(with("dflash2", none, replaced(cfg, "DFlashDraftModel", "DFlash2DraftModel")), - "architecture 'DFlash2DraftModel' is not read by this loader", "a DFlash2 package"); + "no 'vocab_size'", "a first-generation config relabelled DFlash2 is read by the DFlash2 rules, and refused by name"); + refused(with("dflash3", none, replaced(cfg, "DFlashDraftModel", "DFlash3DraftModel")), + "architecture 'DFlash3DraftModel' is not read by this loader", "an architecture this loader does not know"); refused(with("type", none, replaced(cfg, "\"model_type\":\"dflash\"", "\"model_type\":\"qwen3\"")), "model_type is 'qwen3'", "a config that is not a draft's"); refused(with("nokey", none, replaced(cfg, "\"head_dim\":64,", "")), "no 'head_dim'", "a config without head_dim"); @@ -423,6 +425,43 @@ void test_real(const std::string& dir) { } // namespace +// A DFlash 2 package (FastFlowLM/Qwen3.8-27B-DFlash2-NPU2 on this box): every tensor accounted for, one projection of +// each kind decodes finite, and the selector's codebooks give a row each. Decoding the whole draft is 6 GB as f32, so +// this reads one of each instead. +void test_real2(const std::string& dir) { + DraftModel m; + try { + m = load_draft(dir); + } catch (const std::exception& e) { + check(false, std::string("the DFlash 2 package loads: ") + e.what()); + return; + } + const DraftConfig& c = m.cfg; + check(c.generation == 2, "DFlash 2: the generation is read from the architecture"); + check(m.tensors == 3 + 3 + 15 * c.layers, "DFlash 2: every tensor is accounted for (fc, 2 norms, 3 selector, 15 per layer)"); + const auto finite = [](const std::vector& v) { + for (float f : v) + if (!std::isfinite(f)) return false; + return !v.empty(); + }; + check(finite(decode_projection(m.layers[0].q)), "DFlash 2: a Q4_K projection decodes finite"); + check(finite(decode_plain(m.layers[0].attn_conv_base)), "DFlash 2: a convolution base kernel decodes finite"); + std::vector row(size_t(c.selector_rank)); + decode_row(m.selector_succ, int(c.mask_token_id), row.data()); + check(finite(row), "DFlash 2: a codebook row decodes finite"); + bool refused = false; + try { + decode_row(m.selector_succ, c.vocab, row.data()); + } catch (const std::runtime_error&) { + refused = true; + } + check(refused, "DFlash 2: a codebook row past the vocabulary is refused"); + std::printf("DFlash 2 package: %d tensors, %d layers, hidden %d, block %d, taps", m.tensors, c.layers, c.hidden, c.block_size); + for (int l : c.target_layer_ids) std::printf(" %d", l); + std::printf(", conv %d x groups of %d, selector rank %d top %d, vocab %d\n", c.conv_kernel, c.conv_group, c.selector_rank, + c.selector_top_k, c.vocab); +} + int main(int argc, char** argv) { const fs::path root = fs::temp_directory_path() / ("dflash-draft-test-" + std::to_string(::getpid())); fs::remove_all(root); @@ -431,6 +470,9 @@ int main(int argc, char** argv) { const char* real = (argc > 1 && argv[1][0] != '\0') ? argv[1] : std::getenv("ONEBIT_TEST_DFLASH_DRAFT"); if (real != nullptr && *real != '\0') test_real(real); else std::printf("SKIP the real package: pass its directory, or set ONEBIT_TEST_DFLASH_DRAFT\n"); + const char* real2 = (argc > 2 && argv[2][0] != '\0') ? argv[2] : std::getenv("ONEBIT_TEST_DFLASH2_DRAFT"); + if (real2 != nullptr && *real2 != '\0') test_real2(real2); + else std::printf("SKIP the DFlash 2 package: pass its directory second, or set ONEBIT_TEST_DFLASH2_DRAFT\n"); if (g_failures == 0) std::printf("dflash_draft: ok (%d checks)\n", g_checks); return g_failures == 0 ? 0 : 1; } diff --git a/tests/dflash_forward_test.cpp b/tests/dflash_forward_test.cpp index 938b632cd..0ceb3a442 100644 --- a/tests/dflash_forward_test.cpp +++ b/tests/dflash_forward_test.cpp @@ -269,6 +269,27 @@ void properties() { } } +// The context cache: grown in steps it gives exactly the pass draft_forward() gives, and a pass can follow every step. +void cache_steps() { + const DraftWeights w = random_model(5); + Rng r{77}; + const int ctx = 5, block = 3, H = 8; + const std::vector taps = r.vec(size_t(ctx) * 16, 1.0f); + const std::vector noise = r.vec(size_t(block) * size_t(H), 1.0f); + const std::vector fused = fuse_context(w, taps.data(), ctx); + DraftKvCache c; + c.pos0 = 3; + cache_extend(w, c, fused.data(), 2); + const std::vector two = draft_forward_cached(w, c, noise.data(), block); + check(max_diff(two, draft_forward(w, taps.data(), 2, noise.data(), block, 3)) == 0.0, + "cache: two rows, then a pass, is the pass over those two rows"); + cache_extend(w, c, fused.data() + 2 * H, 1); + cache_extend(w, c, fused.data() + 3 * H, 2); + check(c.rows == ctx, "cache: rows add up"); + check(max_diff(draft_forward_cached(w, c, noise.data(), block), draft_forward(w, taps.data(), ctx, noise.data(), block, 3)) == 0.0, + "cache: grown 2 + 1 + 2 rows, it gives exactly draft_forward's pass over all 5"); +} + void refusals() { const DraftWeights w = random_model(3); const std::vector noise(8, 0.5f); @@ -327,10 +348,40 @@ void real_package(const std::string& dir) { } } +// DFlash 2's two extra pieces, each against arithmetic written out here. +void dflash2_pieces() { + { // 3 rows of 4, groups of 2, a kernel of 2 offsets. base[offset][i], dyn[row][offset][group]. + const float x[12] = {1, 2, 3, 4, 5, 6, 7, 8, -1, 0, 2, -2}; + const float base[8] = {1, 1, 1, 1, 0.5f, 0, -1, 2}; // offset 0: identity; offset 1: per element + const float dyn[12] = {0, 0, 9, 9, 1, -1, 0.5f, 0, 0, 2, -1, 1}; + // row 0: only offset 0 exists; (1 + 0) * x0 (its offset-1 kernel, the 9s, has nothing to read) + // row 1: (1 + 1) * 5, (1 + 1) * 6, (1 - 1) * 7, (1 - 1) * 8 + (0.5 + 0.5) * 1, (0 + 0.5) * 2, (-1 + 0) * 3, (2 + 0) * 4 + // row 2: (1 + 0) * -1, (1 + 0) * 0, (1 + 2) * 2, (1 + 2) * -2 + (0.5 - 1) * 5, (0 - 1) * 6, (-1 + 1) * 7, (2 + 1) * 8 + const float want[12] = {1, 2, 3, 4, 10 + 1, 12 + 1, 0 - 3, 0 + 8, -1 - 2.5f, 0 - 6, 6 + 0, -6 + 24}; + const std::vector got = draft_conv(x, 3, 4, 2, 2, base, dyn); + bool same = got.size() == 12; + for (int i = 0; i < 12 && same; i++) same = std::fabs(got[size_t(i)] - want[i]) < 1e-6; + check(same, "DFlash 2 convolution: three rows against the hand-computed ones (a row's own kernel, the row before, " + "the group a value belongs to, nothing before the first row)"); + } + { // 3 candidates, rank 2. scores: 1.0 + (2*0.5*1 + -1*4*0) = 2.0; 1.5 + (2*0.5*-1 + -1*4*0.5) = -1.5; 0.2 + (2*0.5*2 + -1*4*-0.25) = 3.2 + const float unary[3] = {1.0f, 1.5f, 0.2f}, prev[2] = {2.0f, -1.0f}, h[2] = {0.5f, 4.0f}; + const float succ[6] = {1.0f, 0.0f, -1.0f, 0.5f, 2.0f, -0.25f}; + check(selector_pick(unary, 3, prev, h, succ, 2) == 2, + "selector: the pairwise term can lift the lowest-logit candidate over the highest"); + const float flat[6] = {0, 0, 0, 0, 0, 0}; + check(selector_pick(unary, 3, prev, h, flat, 2) == 1, "selector: with no pairwise term it is the plain argmax"); + const float tie[3] = {1.0f, 1.0f, 1.0f}; + check(selector_pick(tie, 3, prev, h, flat, 2) == 0, "selector: the first of equals is kept"); + } +} + } // namespace int main(int argc, char** argv) { hand_cases(); + dflash2_pieces(); + cache_steps(); properties(); refusals(); const char* real = (argc > 1 && argv[1][0] != '\0') ? argv[1] : std::getenv("ONEBIT_TEST_DFLASH_DRAFT");