diff --git a/CMakeLists.txt b/CMakeLists.txt index 00d1d719a..a0b3ce73b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1240,6 +1240,16 @@ endif() # q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks # (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file. add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) +# A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode +# one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which +# lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package, +# so no model, no device; a real package is an optional argument (-DONEBIT_DFLASH_DRAFT=). +add_executable(dflash_draft_test tests/dflash_draft_test.cpp npu/dflash_draft.cpp npu/forward/src/q4nx_dequant.c) +target_include_directories(dflash_draft_test PRIVATE npu npu/forward/include) +target_link_libraries(dflash_draft_test PRIVATE onebit_npu) +add_dependencies(onebit dflash_draft_test) +set(ONEBIT_DFLASH_DRAFT "" CACHE PATH "A DFlash draft package directory for dflash_draft_test's real-package arm") +add_test(NAME dflash_draft COMMAND dflash_draft_test ${ONEBIT_DFLASH_DRAFT}) add_executable(flm_head_import tools/flm_head_import.cpp) target_link_libraries(flm_head_import PRIVATE onebit_npu) add_executable(npu_flm_head_import_test tests/npu_flm_head_import_test.cpp) diff --git a/docs/amd-drafters.md b/docs/amd-drafters.md index 699edf009..a7a7235d1 100644 --- a/docs/amd-drafters.md +++ b/docs/amd-drafters.md @@ -116,6 +116,28 @@ the config, or a tensor it cannot explain (the second draft family, `DFlash2Draf convolution tensors this tool does not know) each fail by name. `--selftest` runs those controls on a synthetic package (ctest `tool_selftest_dflash_draft_info`). +### The loader (`npu/dflash_draft.h`) + +`onebit::npu::dflash::load_draft(dir)` is the C++ half of that gate: it reads the DFlash `config.json`, opens the +container with the engine's own Q4NX reader, checks every tensor against the geometry the config gives, and returns +a model whose tensors are views into the mapped file. Nothing is decoded at load (the Bolt draft's projections are +327,680,000 weights, 1.3 GB as f32); `decode_projection()`, `decode_gain()` and `decode_fusion()` produce one tensor +at a time, through the same q4_1 tile decoder the forward uses. It throws, naming the tensor or key, on a missing or +mis-sized tensor, a dtype label it does not expect, a container scalar that disagrees with the config, a tensor it +cannot explain, another architecture (`DFlash2DraftModel`), or a weight path that leaves the package. + +Two facts about the container that the loader depends on, both measured on the Bolt draft: + +| fact | evidence | +|---|---| +| `model.fc.weight` is **not** row-major. Its declared shape `[80, 12800, 32]` is the element order: 32-row blocks, column by column, the row inside the block last. | Magnitudes that share an input column correlate about five times more strongly than magnitudes that share an output row. Decoded in the declared order the four neighbour figures are consistent (next row 0.107, 32 rows on 0.106; next column 0.022, 32 columns on 0.021). Read as plain row-major they contradict each other (next row 0.019 but 32 rows on 0.107; next column 0.105 but 32 columns on 0.024). `dflash_draft_test` asserts the first and uses the second as its control. | +| A norm gain may be negative. | Three post-attention norms each hold one gain just below zero (-0.0009 to -0.0024) among 2,560. As bf16 that is a gain crossing zero; as half-float the same element would be -0.8 among values of 1.3 to 2.4, which is one more reason the bf16 reading is the right one. | + +`dflash_draft_test` builds a synthetic package (every projection encoded tile by tile from a formula and compared at +every element, the fusion matrix checked for row-major order, sixteen refusals each matched on its message) and needs +no model; given a package directory (`-DONEBIT_DFLASH_DRAFT=` or `ONEBIT_TEST_DFLASH_DRAFT`) it also loads and +decodes the real one. The loader is not wired into `forward-serve` yet: there is no forward for it to feed. + ## PARD-2's repos are weights-only — borrow the target's tokenizer `amd/PARD2-*` ship **`README.md`, `config.json`, `model.safetensors`, `warp_model.bin`** and nothing else: **no tokenizer diff --git a/npu/dflash_draft.cpp b/npu/dflash_draft.cpp new file mode 100644 index 000000000..85a0cec8b --- /dev/null +++ b/npu/dflash_draft.cpp @@ -0,0 +1,275 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// dflash_draft.cpp -- see dflash_draft.h. + +#include "dflash_draft.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "file_map.h" +#include "forward/include/q4nx_dequant.h" +#include "pack_import.h" + +namespace onebit::npu::dflash { + +namespace { + +using nlohmann::json; + +[[noreturn]] void fail(const std::string& what) { throw std::runtime_error("dflash draft: " + what); } + +float bf16(const uint8_t* p) { + uint16_t h; + std::memcpy(&h, p, 2); + const uint32_t bits = uint32_t(h) << 16; + float f; + std::memcpy(&f, &bits, 4); + return f; +} + +int need_int(const json& c, const char* key, const std::string& path) { + if (!c.contains(key)) fail(path + ": no '" + key + "'"); + if (!c[key].is_number_integer()) fail(path + ": '" + key + "' is not an integer"); + const int64_t v = c[key].get(); + if (v <= 0 || v > (int64_t(1) << 30)) fail(path + ": '" + key + "' is " + std::to_string(v)); + return int(v); +} + +float need_float(const json& c, const char* key, const std::string& path) { + if (!c.contains(key)) fail(path + ": no '" + key + "'"); + if (!c[key].is_number()) fail(path + ": '" + key + "' is not a number"); + const double v = c[key].get(); + if (!std::isfinite(v) || v <= 0.0) fail(path + ": '" + key + "' is " + std::to_string(v)); + return float(v); +} + +bool close_to(double a, double b) { return std::fabs(a - b) <= 1e-5 * std::max(1.0, std::fabs(b)); } + +} // namespace + +DraftConfig read_draft_config(const std::string& package_dir) { + const std::string path = package_dir + "/config.json"; + std::ifstream in(path); + if (!in) fail("cannot open " + path); + json c; + try { + in >> c; + } catch (const std::exception& e) { + fail(path + ": not JSON (" + e.what() + ")"); + } + if (!c.is_object()) fail(path + ": not a JSON object"); + const std::string type = c.value("model_type", json("")).is_string() ? c.value("model_type", std::string()) : ""; + if (type != "dflash") fail(path + ": model_type is '" + type + "', not 'dflash'"); + // The architecture decides which tensors exist. This loader knows one; a package naming another is a different + // model (DFlash2 adds a selector and a convolution), and reading it with these rules would drop those silently. + if (c.contains("architectures")) { + if (!c["architectures"].is_array() || c["architectures"].empty() || !c["architectures"][0].is_string()) + fail(path + ": 'architectures' is not a list of names"); + const std::string arch = c["architectures"][0].get(); + if (arch != "DFlashDraftModel") fail(path + ": architecture '" + arch + "' is not read by this loader (DFlashDraftModel is)"); + } + DraftConfig o; + o.hidden = need_int(c, "hidden_size", path); + o.intermediate = need_int(c, "intermediate_size", path); + o.layers = need_int(c, "num_hidden_layers", path); + o.heads = need_int(c, "num_attention_heads", path); + o.kv_heads = need_int(c, "num_key_value_heads", path); + o.head_dim = need_int(c, "head_dim", path); + o.block_size = need_int(c, "block_size", path); + o.sliding_window = need_int(c, "sliding_window", path); + o.rms_eps = need_float(c, "rms_norm_eps", path); + o.rope_theta = need_float(c, "rope_theta", path); + o.embedding_scale = need_float(c, "embedding_scale", path); + if (!c.contains("mask_token_id") || !c["mask_token_id"].is_number_integer() || c["mask_token_id"].get() < 0) + fail(path + ": no usable 'mask_token_id'"); + o.mask_token_id = c["mask_token_id"].get(); + if (!c.contains("target_layer_ids") || !c["target_layer_ids"].is_array() || c["target_layer_ids"].empty()) + fail(path + ": no 'target_layer_ids'"); + int prev = -1; + for (const auto& v : c["target_layer_ids"]) { + if (!v.is_number_integer() || v.get() < 0 || v.get() > 4096) + fail(path + ": 'target_layer_ids' holds something that is not a layer index"); + const int id = int(v.get()); + if (id <= prev) fail(path + ": 'target_layer_ids' is not strictly increasing at " + std::to_string(id)); + o.target_layer_ids.push_back(id); + prev = id; + } + if (!c.contains("draft_weight") || !c["draft_weight"].is_string() || c["draft_weight"].get().empty()) + fail(path + ": no 'draft_weight'"); + o.draft_weight = c["draft_weight"].get(); + if (o.draft_weight.find('/') != std::string::npos || o.draft_weight.find("..") != std::string::npos) + fail(path + ": 'draft_weight' must be a file name inside the package, got '" + o.draft_weight + "'"); + if (c.contains("causal")) { + if (!c["causal"].is_boolean()) fail(path + ": 'causal' is not a boolean"); + o.causal = c["causal"].get(); + } + if (c.contains("base_model") && c["base_model"].is_string()) o.base_model = c["base_model"].get(); + if (o.kv_heads > o.heads || o.heads % o.kv_heads != 0) + fail(path + ": " + std::to_string(o.heads) + " heads cannot share " + std::to_string(o.kv_heads) + " key/value heads"); + if (o.block_size < 2) fail(path + ": block_size " + std::to_string(o.block_size) + " drafts nothing"); + return o; +} + +DraftModel load_draft(const std::string& package_dir) { + DraftModel m; + m.cfg = read_draft_config(package_dir); + const DraftConfig& c = m.cfg; + const std::string path = package_dir + "/" + c.draft_weight; + + const pack_import::Q4nxContainer table = pack_import::read_q4nx_container(path); + if (!table.why.empty()) fail(table.why); + size_t size = 0; + std::string why; + const uint8_t* base = map_file(path, size, &why); + if (base == nullptr) fail(why); + m.keep = std::shared_ptr(base, [size](const void* p) { unmap_file(static_cast(p), size); }); + m.container_bytes = size; + m.tensors = int(table.tensors.size()); + const size_t payload = 8 + size_t(table.header_bytes); + + std::map by_name; + for (const auto& t : table.tensors) by_name[t.name] = &t; + std::set used; + + // Every tensor is taken through here, so "inside the file" and "accounted for" are checked once, for all of them. + const auto take = [&](const std::string& name, const char* dtype, size_t want_bytes, int rows, int cols) { + const auto it = by_name.find(name); + if (it == by_name.end()) fail(path + ": no tensor '" + name + "'"); + const pack_import::Q4nxTensor& t = *it->second; + if (t.dtype != dtype) fail(path + ": '" + name + "' is declared " + t.dtype + ", expected " + dtype); + if (t.begin < 0 || t.end < t.begin || payload + size_t(t.end) > size) + fail(path + ": '" + name + "' lies outside the file"); + const size_t have = size_t(t.end - t.begin); + if (have != want_bytes) + fail(path + ": '" + name + "' holds " + std::to_string(have) + " B, and " + std::to_string(rows) + " x " + + std::to_string(cols) + " needs " + std::to_string(want_bytes)); + used.insert(name); + TensorView v; + v.name = name; + v.data = base + payload + size_t(t.begin); + v.bytes = have; + v.rows = rows; + v.cols = cols; + return v; + }; + const auto projection = [&](const std::string& name, int n, int k) { + if (n % 32 != 0 || k % 256 != 0) + fail(path + ": '" + name + "' would be " + std::to_string(n) + " x " + std::to_string(k) + + ", which is not a whole 32 x 256 tile grid"); + return take(name, "I8", q4nx_tensor_bytes(n, k), n, k); + }; + const auto gain = [&](const std::string& name, int width) { return take(name, "BF16", size_t(width) * 2, 1, width); }; + + // The container repeats the interface scalars; a package whose two halves disagree is not one model. + const auto scalar = [&](const std::string& name, const char* dtype, size_t count) { + return take(name, dtype, count * 4, 1, int(count)).data; + }; + const auto i32_at = [](const uint8_t* p, size_t i) { int32_t v; std::memcpy(&v, p + i * 4, 4); return v; }; + const auto f32_at = [](const uint8_t* p) { float v; std::memcpy(&v, p, 4); return v; }; + const auto agree = [&](bool same, const std::string& name, const std::string& got, const std::string& want) { + if (!same) fail(path + ": '" + name + "' is " + got + " and config.json says " + want); + }; + { + const uint8_t* p = scalar("dflash.block_size", "I32", 1); + agree(i32_at(p, 0) == c.block_size, "dflash.block_size", std::to_string(i32_at(p, 0)), std::to_string(c.block_size)); + p = scalar("dflash.sliding_window", "I32", 1); + agree(i32_at(p, 0) == c.sliding_window, "dflash.sliding_window", std::to_string(i32_at(p, 0)), std::to_string(c.sliding_window)); + p = scalar("dflash.target_layer_ids", "I32", size_t(c.taps())); + for (int i = 0; i < c.taps(); ++i) + agree(i32_at(p, size_t(i)) == c.target_layer_ids[size_t(i)], "dflash.target_layer_ids[" + std::to_string(i) + "]", + std::to_string(i32_at(p, size_t(i))), std::to_string(c.target_layer_ids[size_t(i)])); + p = scalar("dflash.rope_theta", "F32", 1); + agree(close_to(f32_at(p), c.rope_theta), "dflash.rope_theta", std::to_string(f32_at(p)), std::to_string(c.rope_theta)); + p = scalar("dflash.rms_eps", "F32", 1); + agree(close_to(f32_at(p) * 1e6, double(c.rms_eps) * 1e6), "dflash.rms_eps", std::to_string(f32_at(p)), std::to_string(c.rms_eps)); + p = scalar("dflash.embedding_scale", "F32", 1); + agree(close_to(f32_at(p), c.embedding_scale), "dflash.embedding_scale", std::to_string(f32_at(p)), std::to_string(c.embedding_scale)); + } + + // The fusion matrix: hidden x (taps * hidden), stored in 32-row blocks (see decode_fusion), so hidden must fill them + // and the declared shape must be that blocking -- a plain [hidden, taps*hidden] label would mean another element order. + if (c.hidden % 32 != 0) fail(path + ": hidden_size " + std::to_string(c.hidden) + " does not fill 32-row blocks"); + const int fused = c.taps() * c.hidden; + m.fc = take("model.fc.weight", "BF16", size_t(c.hidden) * size_t(fused) * 2, c.hidden, fused); + { + const std::vector want = {c.hidden / 32, fused, 32}; + if (by_name["model.fc.weight"]->shape != want) + fail(path + ": 'model.fc.weight' is not declared [hidden/32, taps*hidden, 32], so its element order is not the one this loader undoes"); + } + m.hidden_norm = gain("model.hidden_norm.weight", c.hidden); + m.norm = gain("model.norm.weight", c.hidden); + for (int l = 0; l < c.layers; ++l) { + const std::string p = "model.layers." + std::to_string(l) + "."; + DraftLayer L; + L.q = projection(p + "self_attn.q_proj.weight", c.q_dim(), c.hidden); + L.k = projection(p + "self_attn.k_proj.weight", c.kv_dim(), c.hidden); + L.v = projection(p + "self_attn.v_proj.weight", c.kv_dim(), c.hidden); + L.o = projection(p + "self_attn.o_proj.weight", c.hidden, c.q_dim()); + L.gate = projection(p + "mlp.gate_proj.weight", c.intermediate, c.hidden); + L.up = projection(p + "mlp.up_proj.weight", c.intermediate, c.hidden); + L.down = projection(p + "mlp.down_proj.weight", c.hidden, c.intermediate); + L.input_norm = gain(p + "input_layernorm.weight", c.hidden); + L.post_norm = gain(p + "post_attention_layernorm.weight", c.hidden); + L.q_norm = gain(p + "self_attn.q_norm.weight", c.head_dim); + L.k_norm = gain(p + "self_attn.k_norm.weight", c.head_dim); + m.layers.push_back(L); + } + for (const auto& t : table.tensors) + if (used.count(t.name) == 0) + fail(path + ": tensor '" + t.name + "' is not part of a DFlashDraftModel (" + std::to_string(table.tensors.size() - used.size()) + + " unexplained): this is another architecture"); + return m; +} + +std::vector decode_projection(const TensorView& t) { + if (!t.ok()) fail("decode_projection of an empty view"); + std::vector out(size_t(t.rows) * size_t(t.cols)); + // Not the size-dispatching q4nx_dequant_tensor(): a projection here is a tile grid by contract, and a grid whose + // byte count happened to equal rows*cols*2 must not be read as plain 2-byte floats. + if (t.bytes != q4nx_tensor_bytes(t.rows, t.cols) || + q4nx_dequant_tensor_chunked(t.data, t.bytes, t.rows, t.cols, 5120, out.data()) != 0) + fail("'" + t.name + "' does not decode as a " + std::to_string(t.rows) + " x " + std::to_string(t.cols) + " q4_1 tile grid"); + return out; +} + +std::vector decode_gain(const TensorView& t) { + if (!t.ok() || t.bytes != size_t(t.cols) * 2) fail("decode_gain of '" + t.name + "': not 2 bytes per element"); + std::vector out(size_t(t.cols)); + for (int i = 0; i < t.cols; ++i) out[size_t(i)] = bf16(t.data + size_t(i) * 2); + return out; +} + +std::vector decode_fusion(const TensorView& t) { + if (!t.ok() || t.rows % 32 != 0 || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) + fail("decode_fusion of '" + t.name + "': not hidden x fused at 2 bytes per element"); + const size_t cols = size_t(t.cols); + std::vector out(size_t(t.rows) * cols); + for (size_t a = 0; a < size_t(t.rows) / 32; ++a) + for (size_t k = 0; k < cols; ++k) { + const uint8_t* p = t.data + ((a * cols + k) * 32) * 2; + for (size_t r = 0; r < 32; ++r) out[(a * 32 + r) * cols + k] = bf16(p + r * 2); + } + return out; +} + +} // namespace onebit::npu::dflash diff --git a/npu/dflash_draft.h b/npu/dflash_draft.h new file mode 100644 index 000000000..d5e72ad51 --- /dev/null +++ b/npu/dflash_draft.h @@ -0,0 +1,113 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// dflash_draft.h -- load a DFlash draft package as a model source (engine#1780, the drafts half of F2). +// +// A draft is a two-file source: `config.json` in the DFlash schema (model_type "dflash": the draft's own geometry, the +// block size, the mask token, the target layers it reads) and a Q4NX container with its weights. The container is in +// forms this tree already decodes (tools/dflash_draft_info.py is the gate that established it), so this is a loader +// and not a format: it checks every tensor against the geometry the config gives, and hands each one out decoded. +// +// What it guarantees, and refuses by name otherwise: +// - the config is the DFlash schema with every key the loader needs; another architecture (DFlash2, with selector +// and convolution tensors) is refused rather than half-read; +// - every projection of every layer is present and is exactly the q4_1 tile grid of the N x K the config implies; +// - every norm and the fusion matrix are present at 2 bytes per element in the declared shape; +// - the container's own scalars agree with config.json; +// - no tensor is left unexplained: a tensor this loader does not know means an architecture it does not know. +// +// Nothing is decoded at load. A draft's projections are 1.3 GB as f32, so the model holds views into the mapped file +// and decode_*() produces one tensor at a time; the caller decides what to keep. +// +// Not here: the draft's forward (five non-causal layers over a masked block), the target-side taps, and anything on +// the device. This is the step before them. + +#pragma once + +#include +#include +#include +#include +#include + +namespace onebit::npu::dflash { + +struct DraftConfig { + int hidden = 0; // hidden_size: the draft's width, and the target model's + int intermediate = 0; // intermediate_size + int layers = 0; // num_hidden_layers + int heads = 0; // num_attention_heads + int kv_heads = 0; // num_key_value_heads + int head_dim = 0; + int block_size = 0; // positions per drafted block (the first is the last accepted token) + int64_t mask_token_id = -1; + std::vector target_layer_ids; // the target layers whose hidden states the draft reads, in fusion order + int sliding_window = 0; + float rms_eps = 0.0f; + float rope_theta = 0.0f; + float embedding_scale = 0.0f; + bool causal = false; + std::string draft_weight; // the container's file name, relative to the package + std::string base_model; // the target this draft was made for, as the package names it (may be empty) + + int taps() const { return int(target_layer_ids.size()); } + int q_dim() const { return heads * head_dim; } + int kv_dim() const { return kv_heads * head_dim; } +}; + +// One tensor of the container: where its bytes are and the matrix they describe. +struct TensorView { + std::string name; + const uint8_t* data = nullptr; + size_t bytes = 0; + int rows = 0; // N (out); 1 for a vector + int cols = 0; // K (in), or the vector's length + bool ok() const { return data != nullptr && bytes != 0; } +}; + +struct DraftLayer { + TensorView q, k, v, o, gate, up, down; // q4_1 tile grids, N x K + TensorView input_norm, post_norm, q_norm, k_norm; // 2-byte gains +}; + +struct DraftModel { + DraftConfig cfg; + TensorView fc; // hidden x (taps * hidden): fuses the target layers' hidden states + TensorView hidden_norm; // applied to the fused features + TensorView norm; // the final norm + std::vector layers; + size_t container_bytes = 0; + int tensors = 0; // tensors in the container, all of them accounted for + std::shared_ptr keep; // owns the mapping every view points into +}; + +// config.json of a draft package. Throws std::runtime_error naming the key or the value that is wrong. +DraftConfig read_draft_config(const std::string& package_dir); + +// The whole package: config, container, every check above. Throws std::runtime_error with the tensor and the reason. +DraftModel load_draft(const std::string& package_dir); + +// A projection as f32, row-major [rows][cols] (out[n * cols + k] multiplies input k into output n). +std::vector decode_projection(const TensorView& t); + +// A norm's gains as f32. +std::vector decode_gain(const TensorView& t); + +// The fusion matrix as f32, row-major [hidden][taps * hidden]. The container stores it as [hidden/32][taps*hidden][32] +// -- 32-row blocks, column by column, the row inside the block last -- and that is the order this undoes: +// out[(a*32 + r) * cols + k] = stored[(a * cols + k) * 32 + r]. +std::vector decode_fusion(const TensorView& t); + +} // namespace onebit::npu::dflash diff --git a/tests/dflash_draft_test.cpp b/tests/dflash_draft_test.cpp new file mode 100644 index 000000000..52735cd82 --- /dev/null +++ b/tests/dflash_draft_test.cpp @@ -0,0 +1,436 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// dflash_draft_test.cpp -- the DFlash draft loader (npu/dflash_draft.h, engine#1780). +// +// Two halves: +// 1. a synthetic package written here, small enough to build in a moment, with known answers: every projection is +// encoded tile by tile from a formula and must decode to that formula's values; the fusion matrix must come back +// in row-major order; and each way a package can be wrong is refused with a message naming the thing; +// 2. the real Bolt draft, when a path is given (argv[1] or ONEBIT_TEST_DFLASH_DRAFT): every tensor loads and decodes +// finite, one at a time, and the fusion matrix's element order is checked against a property of the weights +// themselves (see fusion_order_holds). Without a path this half prints SKIP and the test still passes. +// +// No device, no kernels, no forward. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dflash_draft.h" + +namespace { + +namespace fs = std::filesystem; +using namespace onebit::npu::dflash; + +int g_failures = 0; +int g_checks = 0; + +void check(bool cond, const std::string& what) { + ++g_checks; + if (!cond) { + std::printf("FAIL %s\n", what.c_str()); + ++g_failures; + } +} + +uint16_t to_bf16(float f) { // exact for the values used here (few mantissa bits) + uint32_t bits; + std::memcpy(&bits, &f, 4); + return uint16_t(bits >> 16); +} +void put16(std::string& s, uint16_t v) { s.append(reinterpret_cast(&v), 2); } +void put32(std::string& s, const void* p) { s.append(static_cast(p), 4); } + +// The synthetic weights, as formulas a decoder has to reproduce. +int code_of(int salt, int n, int k) { return (n * 7 + k * 3 + salt) & 15; } +float scale_of(int salt, int n, int group) { return 0.0078125f * float(1 + ((n + group + salt) % 4)); } // 1/128 .. 4/128 +float min_of(int salt, int n, int group) { return -0.0625f * float(1 + ((n * 3 + group + salt) % 5)); } +float weight_of(int salt, int n, int k) { + return float(code_of(salt, n, k)) * scale_of(salt, n, k / 32) + min_of(salt, n, k / 32); +} +float fusion_of(int n, int k) { return 0.03125f * float((n * 5 + k * 11) % 64 - 32); } +float gain_of(int salt, int i) { return 1.0f + 0.125f * float((i + salt) % 8); } + +// One N x K projection as a q4_1 tile grid: tiles row-major, each 5120 B = 256 bf16 scales (index group*32 + row), +// 256 bf16 minimums (same index), then 4096 B of 4-bit codes (two 16-row halves x 8 groups, 256 B each; inside one, +// byte 8*j + row/2 for column j of the group, even row in the low nibble). +std::string encode_projection(int salt, int N, int K) { + std::string out; + for (int tr = 0; tr < N / 32; ++tr) + for (int tc = 0; tc < K / 256; ++tc) { + std::string tile(5120, '\0'); + for (int g = 0; g < 8; ++g) + for (int r = 0; r < 32; ++r) { + const int n = tr * 32 + r, group = tc * 8 + g; + const uint16_t s = to_bf16(scale_of(salt, n, group)), z = to_bf16(min_of(salt, n, group)); + std::memcpy(&tile[size_t(2 * (g * 32 + r))], &s, 2); + std::memcpy(&tile[size_t(512 + 2 * (g * 32 + r))], &z, 2); + } + for (int r = 0; r < 32; ++r) + for (int c = 0; c < 256; ++c) { + const int half = r / 16, rr = r % 16, g = c / 32, j = c % 32; + const int code = code_of(salt, tr * 32 + r, tc * 256 + c); + unsigned char& b = reinterpret_cast(tile[size_t(1024 + (half * 8 + g) * 256 + 8 * j + rr / 2)]); + b = (rr % 2 == 0) ? (unsigned char)((b & 0xF0) | code) : (unsigned char)((b & 0x0F) | (code << 4)); + } + out += tile; + } + return out; +} + +struct Geometry { + int hidden = 256, intermediate = 512, layers = 2, heads = 4, kv_heads = 2, head_dim = 64, block_size = 8; + std::vector taps = {1, 3}; +}; + +struct Entry { + std::string name, dtype, shape, bytes; +}; + +// The package's tensors in container order. `edit` may rewrite the list (drop one, add one, damage one). +std::vector tensors_of(const Geometry& g) { + std::vector t; + const auto scalar_f = [&](const std::string& n, float v) { std::string b; put32(b, &v); t.push_back({n, "F32", "[1]", b}); }; + const auto scalar_i = [&](const std::string& n, int32_t v) { std::string b; put32(b, &v); t.push_back({n, "I32", "[1]", b}); }; + scalar_f("dflash.embedding_scale", 16.0f); + scalar_f("dflash.rms_eps", 1e-6f); + scalar_f("dflash.rope_theta", 1000000.0f); + scalar_i("dflash.block_size", g.block_size); + scalar_i("dflash.sliding_window", 1024); + { + std::string b; + for (int id : g.taps) { const int32_t v = id; put32(b, &v); } + t.push_back({"dflash.target_layer_ids", "I32", "[" + std::to_string(g.taps.size()) + "]", b}); + } + const int fused = int(g.taps.size()) * g.hidden; + { + std::string b; // [hidden/32][fused][32] + for (int a = 0; a < g.hidden / 32; ++a) + for (int k = 0; k < fused; ++k) + for (int r = 0; r < 32; ++r) put16(b, to_bf16(fusion_of(a * 32 + r, k))); + t.push_back({"model.fc.weight", "BF16", + "[" + std::to_string(g.hidden / 32) + "," + std::to_string(fused) + ",32]", b}); + } + const auto gain = [&](const std::string& n, int salt, int width) { + std::string b; + for (int i = 0; i < width; ++i) put16(b, to_bf16(gain_of(salt, i))); + t.push_back({n, "BF16", "[" + std::to_string(width) + "]", b}); + }; + const auto proj = [&](const std::string& n, int salt, int N, int K) { + const std::string b = encode_projection(salt, N, K); + t.push_back({n, "I8", "[" + std::to_string(b.size() / 5120) + ",5120]", b}); + }; + gain("model.hidden_norm.weight", 1, g.hidden); + const int q = g.heads * g.head_dim, kv = g.kv_heads * g.head_dim; + for (int l = 0; l < g.layers; ++l) { + const std::string p = "model.layers." + std::to_string(l) + "."; + const int s = 10 * (l + 1); + gain(p + "input_layernorm.weight", s + 1, g.hidden); + gain(p + "post_attention_layernorm.weight", s + 2, g.hidden); + gain(p + "self_attn.k_norm.weight", s + 3, g.head_dim); + gain(p + "self_attn.q_norm.weight", s + 4, g.head_dim); + proj(p + "self_attn.q_proj.weight", s + 1, q, g.hidden); + proj(p + "self_attn.k_proj.weight", s + 2, kv, g.hidden); + proj(p + "self_attn.v_proj.weight", s + 3, kv, g.hidden); + proj(p + "self_attn.o_proj.weight", s + 4, g.hidden, q); + proj(p + "mlp.gate_proj.weight", s + 5, g.intermediate, g.hidden); + proj(p + "mlp.up_proj.weight", s + 6, g.intermediate, g.hidden); + proj(p + "mlp.down_proj.weight", s + 7, g.hidden, g.intermediate); + } + gain("model.norm.weight", 2, g.hidden); + return t; +} + +std::string config_of(const Geometry& g) { + std::string taps; + for (size_t i = 0; i < g.taps.size(); ++i) taps += (i ? "," : "") + std::to_string(g.taps[i]); + return "{\"model_type\":\"dflash\",\"architectures\":[\"DFlashDraftModel\"],\"base_model\":\"synthetic\"," + "\"draft_weight\":\"dflash_weight.q4nx\",\"block_size\":" + std::to_string(g.block_size) + + ",\"mask_token_id\":4,\"target_layer_ids\":[" + taps + "],\"num_hidden_layers\":" + std::to_string(g.layers) + + ",\"hidden_size\":" + std::to_string(g.hidden) + ",\"intermediate_size\":" + std::to_string(g.intermediate) + + ",\"num_attention_heads\":" + std::to_string(g.heads) + ",\"num_key_value_heads\":" + std::to_string(g.kv_heads) + + ",\"head_dim\":" + std::to_string(g.head_dim) + + ",\"rms_norm_eps\":1e-06,\"rope_theta\":1000000.0,\"sliding_window\":1024,\"embedding_scale\":16.0,\"causal\":false}"; +} + +void write_package(const fs::path& dir, const std::string& config, const std::vector& tensors, size_t drop_tail = 0) { + fs::create_directories(dir); + std::ofstream(dir / "config.json") << config; + std::string header = "{\"__metadata__\":{}", payload; + for (const Entry& e : tensors) { + header += ",\"" + e.name + "\":{\"dtype\":\"" + e.dtype + "\",\"shape\":" + e.shape + ",\"data_offsets\":[" + + std::to_string(payload.size()) + "," + std::to_string(payload.size() + e.bytes.size()) + "]}"; + payload += e.bytes; + } + header += "}"; + const uint64_t n = header.size(); + std::ofstream f(dir / "dflash_weight.q4nx", std::ios::binary); + f.write(reinterpret_cast(&n), 8); + f.write(header.data(), std::streamsize(header.size())); + f.write(payload.data(), std::streamsize(payload.size() - drop_tail)); +} + +std::string replaced(std::string s, const std::string& from, const std::string& to) { + const size_t at = s.find(from); + if (at == std::string::npos) { + std::printf("FAIL test bug: '%s' is not in the config\n", from.c_str()); + ++g_failures; + return s; + } + return s.replace(at, from.size(), to); +} + +// load_draft() must throw, and the message must name `needle`. +void refused(const fs::path& dir, const std::string& needle, const std::string& what) { + try { + (void)load_draft(dir.string()); + check(false, what + ": accepted"); + } catch (const std::runtime_error& e) { + const bool named = std::string(e.what()).find(needle) != std::string::npos; + check(named, what + ": refused, but the message does not say '" + needle + "': " + e.what()); + } +} + +void test_synthetic(const fs::path& root) { + const Geometry g; + const std::vector good = tensors_of(g); + const std::string cfg = config_of(g); + + { // the package as it should be + write_package(root / "good", cfg, good); + DraftModel m; + try { + m = load_draft((root / "good").string()); + } catch (const std::exception& e) { + check(false, std::string("the synthetic package loads: ") + e.what()); + return; + } + check(m.cfg.hidden == 256 && m.cfg.intermediate == 512 && m.cfg.layers == 2 && m.cfg.heads == 4 && m.cfg.kv_heads == 2 && + m.cfg.head_dim == 64 && m.cfg.block_size == 8 && m.cfg.mask_token_id == 4 && !m.cfg.causal, + "the config's geometry is read"); + check(m.cfg.target_layer_ids == std::vector({1, 3}) && m.cfg.taps() == 2 && m.cfg.base_model == "synthetic", + "and its target layers, in order"); + check(m.layers.size() == 2 && m.tensors == int(good.size()) && m.tensors == 6 + 3 + 2 * 11, "every tensor is accounted for"); + check(m.fc.rows == 256 && m.fc.cols == 512, "the fusion matrix is hidden x (taps * hidden)"); + + const auto same = [&](const TensorView& v, int salt, int N, int K, const std::string& name) { + check(v.rows == N && v.cols == K, name + " is " + std::to_string(N) + " x " + std::to_string(K)); + const std::vector w = decode_projection(v); + size_t bad = 0; + for (int n = 0; n < N; ++n) + for (int k = 0; k < K; ++k) + if (w[size_t(n) * size_t(K) + size_t(k)] != weight_of(salt, n, k)) ++bad; + check(w.size() == size_t(N) * size_t(K) && bad == 0, + name + " decodes to the encoded values at every element (" + std::to_string(bad) + " differ)"); + }; + for (int l = 0; l < 2; ++l) { + const DraftLayer& L = m.layers[size_t(l)]; + const int s = 10 * (l + 1); + const std::string p = "layer " + std::to_string(l) + " "; + same(L.q, s + 1, 256, 256, p + "q_proj"); + same(L.k, s + 2, 128, 256, p + "k_proj"); + same(L.v, s + 3, 128, 256, p + "v_proj"); + same(L.o, s + 4, 256, 256, p + "o_proj"); + same(L.gate, s + 5, 512, 256, p + "gate_proj"); + same(L.up, s + 6, 512, 256, p + "up_proj"); + same(L.down, s + 7, 256, 512, p + "down_proj"); + const std::vector in = decode_gain(L.input_norm), qn = decode_gain(L.q_norm), kn = decode_gain(L.k_norm), + post = decode_gain(L.post_norm); + bool ok = in.size() == 256 && post.size() == 256 && qn.size() == 64 && kn.size() == 64; + for (size_t i = 0; ok && i < in.size(); ++i) ok = in[i] == gain_of(s + 1, int(i)) && post[i] == gain_of(s + 2, int(i)); + for (size_t i = 0; ok && i < qn.size(); ++i) ok = kn[i] == gain_of(s + 3, int(i)) && qn[i] == gain_of(s + 4, int(i)); + check(ok, p + "norm gains decode to the encoded values, each under its own name"); + } + const std::vector fc = decode_fusion(m.fc); + size_t bad = 0; + for (int n = 0; n < 256; ++n) + for (int k = 0; k < 512; ++k) + if (fc[size_t(n) * 512 + size_t(k)] != fusion_of(n, k)) ++bad; + check(fc.size() == 256u * 512u && bad == 0, "the fusion matrix comes back row-major (" + std::to_string(bad) + " elements differ)"); + // and the stored order really is a different one: read plainly, element (1, 0) is not what is stored second + check(fusion_of(1, 0) != fusion_of(0, 1), "the fixture distinguishes the two element orders"); + const std::vector hn = decode_gain(m.hidden_norm), fn = decode_gain(m.norm); + check(hn[3] == gain_of(1, 3) && fn[3] == gain_of(2, 3) && hn[3] != fn[3], "hidden_norm and the final norm are not swapped"); + } + + // What must be refused, each by name. + const auto with = [&](const std::string& name, const std::function&)>& edit, const std::string& config) { + std::vector t = good; + edit(t); + write_package(root / name, config, t); + return root / name; + }; + const auto none = [](std::vector&) {}; + const auto find = [](std::vector& t, const std::string& n) -> Entry& { + for (Entry& e : t) + if (e.name == n) return e; + std::printf("FAIL test bug: no tensor %s\n", n.c_str()); + ++g_failures; + return t[0]; + }; + refused(with("missing", [&](std::vector& t) { + for (size_t i = 0; i < t.size(); ++i) + if (t[i].name == "model.layers.1.mlp.up_proj.weight") { t.erase(t.begin() + long(i)); break; } + }, cfg), + "no tensor 'model.layers.1.mlp.up_proj.weight'", "a missing projection"); + refused(with("short", [&](std::vector& t) { find(t, "model.layers.0.self_attn.k_proj.weight").bytes.resize(5120); }, cfg), + "'model.layers.0.self_attn.k_proj.weight' holds 5120 B", "a projection with too few tiles"); + refused(with("extra", [&](std::vector& t) { t.push_back({"model.selector.weight", "BF16", "[4]", std::string(8, '\0')}); }, cfg), + "'model.selector.weight' is not part of a DFlashDraftModel", "a tensor of another architecture"); + refused(with("dtype", [&](std::vector& t) { find(t, "model.norm.weight").dtype = "F16"; }, cfg), + "'model.norm.weight' is declared F16", "a gain under another dtype label"); + refused(with("plainfc", [&](std::vector& t) { find(t, "model.fc.weight").shape = "[256,512]"; }, cfg), + "'model.fc.weight' is not declared [hidden/32, taps*hidden, 32]", "a fusion matrix declared in another element order"); + refused(with("block", [&](std::vector& t) { const int32_t v = 16; std::memcpy(find(t, "dflash.block_size").bytes.data(), &v, 4); }, cfg), + "'dflash.block_size' is 16 and config.json says 8", "a container that disagrees with its config on the block size"); + refused(with("taps", [&](std::vector& t) { const int32_t v = 2; std::memcpy(find(t, "dflash.target_layer_ids").bytes.data() + 4, &v, 4); }, cfg), + "'dflash.target_layer_ids[1]' is 2 and config.json says 3", "a container that disagrees on a target layer"); + refused(with("dflash2", none, replaced(cfg, "DFlashDraftModel", "DFlash2DraftModel")), + "architecture 'DFlash2DraftModel' is not read by this loader", "a DFlash2 package"); + refused(with("type", none, replaced(cfg, "\"model_type\":\"dflash\"", "\"model_type\":\"qwen3\"")), + "model_type is 'qwen3'", "a config that is not a draft's"); + refused(with("nokey", none, replaced(cfg, "\"head_dim\":64,", "")), "no 'head_dim'", "a config without head_dim"); + refused(with("order", none, replaced(cfg, "[1,3]", "[3,1]")), "'target_layer_ids' is not strictly increasing", "target layers out of order"); + refused(with("path", none, replaced(cfg, "\"dflash_weight.q4nx\"", "\"../good/dflash_weight.q4nx\"")), + "'draft_weight' must be a file name inside the package", "a weight path that leaves the package"); + refused(with("grid", none, replaced(cfg, "\"intermediate_size\":512", "\"intermediate_size\":500")), + "is not a whole 32 x 256 tile grid", "a geometry that does not fill whole tiles"); + refused(with("heads", none, replaced(cfg, "\"num_key_value_heads\":2", "\"num_key_value_heads\":3")), + "4 heads cannot share 3 key/value heads", "key/value heads that do not divide the heads"); + { // the last tensor's bytes are not all there + write_package(root / "cut", cfg, good, 100); + refused(root / "cut", "", "a container cut short of its last tensor"); + } + { + fs::create_directories(root / "empty"); + refused(root / "empty", "cannot open", "a directory with no config.json"); + } +} + +// The fusion matrix's stored element order, checked on the weights themselves. Magnitudes that share an input column +// are correlated far more strongly than magnitudes that share an output row (each column multiplies one feature of a +// target layer, and features differ in scale). In the right order that holds whichever neighbours are compared: the next +// row and the row 32 away agree with each other, and so do the next column and the column 32 away. Read in the plain +// row-major order instead, "the next column" is really the same column and "32 columns away" is not, so the two +// column figures disagree by several times. The assertion is that consistency, not a threshold on one number. +bool fusion_order_holds(const std::vector& w, int rows, int cols, double out[4]) { + std::vector lw(w.size()); + double mean = 0.0; + for (size_t i = 0; i < w.size(); ++i) { + lw[i] = std::log(std::fabs(w[i]) + 1e-6f); + mean += lw[i]; + } + mean /= double(w.size()); + double var = 0.0; + for (float v : lw) var += (double(v) - mean) * (double(v) - mean); + var /= double(w.size()); + const auto corr = [&](int dn, int dk) { + double acc = 0.0; + size_t count = 0; + for (int n = 0; n + dn < rows; n += 3) + for (int k = 0; k + dk < cols; k += 7) { + acc += (double(lw[size_t(n) * size_t(cols) + size_t(k)]) - mean) * + (double(lw[size_t(n + dn) * size_t(cols) + size_t(k + dk)]) - mean); + ++count; + } + return acc / double(count) / var; + }; + out[0] = corr(1, 0); // next row, same column + out[1] = corr(32, 0); // 32 rows on, same column + out[2] = corr(0, 1); // same row, next column + out[3] = corr(0, 32); // same row, 32 columns on + const auto near = [](double a, double b) { return std::fabs(a - b) <= 0.5 * std::max(std::fabs(a), std::fabs(b)); }; + return near(out[0], out[1]) && near(out[2], out[3]) && out[0] > 2.0 * std::max(out[2], out[3]); +} + +void test_real(const std::string& dir) { + DraftModel m; + try { + m = load_draft(dir); + } catch (const std::exception& e) { + check(false, std::string("the real package loads: ") + e.what()); + return; + } + const DraftConfig& c = m.cfg; + std::printf("real package: %d tensors, %zu bytes; %d layers, hidden %d, intermediate %d, %d heads / %d kv x %d, block %d, taps", + m.tensors, m.container_bytes, c.layers, c.hidden, c.intermediate, c.heads, c.kv_heads, c.head_dim, c.block_size); + for (int id : c.target_layer_ids) std::printf(" %d", id); + std::printf("\n"); + check(int(m.layers.size()) == c.layers && m.tensors == 6 + 3 + 11 * c.layers, "every tensor of the real package is accounted for"); + size_t elements = 0; + for (const DraftLayer& L : m.layers) { + for (const TensorView* v : {&L.q, &L.k, &L.v, &L.o, &L.gate, &L.up, &L.down}) { + const std::vector w = decode_projection(*v); // one at a time: the set is 1.3 GB as f32 + size_t nonfinite = 0; + double sum2 = 0.0; + for (float x : w) { + if (!std::isfinite(x)) ++nonfinite; + else sum2 += double(x) * double(x); + } + const double rms = std::sqrt(sum2 / double(w.size())); + check(nonfinite == 0 && rms > 1e-4 && rms < 1.0, + v->name + " decodes finite with a weight-like rms (" + std::to_string(rms) + ", " + std::to_string(nonfinite) + " non-finite)"); + elements += w.size(); + } + for (const TensorView* v : {&L.input_norm, &L.post_norm, &L.q_norm, &L.k_norm}) { + // Finite and bounded, not "positive": three of the Bolt draft's post-attention norms hold one gain just + // below zero (-0.0009 .. -0.0024), which is what a learned gain crossing zero looks like. + bool ok = true; + for (float x : decode_gain(*v)) ok = ok && std::isfinite(x) && std::fabs(x) < 100.0f; + check(ok, v->name + " is finite and bounded"); + } + } + std::printf("real package: %zu projection weights decoded\n", elements); + const std::vector fc = decode_fusion(m.fc); + double r[4]; + const bool right = fusion_order_holds(fc, m.fc.rows, m.fc.cols, r); + std::printf("fusion matrix, as decoded: next row %.4f, 32 rows on %.4f | next column %.4f, 32 columns on %.4f\n", r[0], r[1], r[2], r[3]); + check(right, "the fusion matrix's decoded order is self-consistent: column-sharing neighbours agree, row-sharing neighbours agree"); + // the control: the same bytes read as plain row-major must NOT pass, or the check above proves nothing + std::vector plain(fc.size()); + for (size_t i = 0; i < plain.size(); ++i) { + uint16_t h; + std::memcpy(&h, m.fc.data + i * 2, 2); + const uint32_t bits = uint32_t(h) << 16; + std::memcpy(&plain[i], &bits, 4); + } + const bool wrong = fusion_order_holds(plain, m.fc.rows, m.fc.cols, r); + std::printf("fusion matrix, read plainly: next row %.4f, 32 rows on %.4f | next column %.4f, 32 columns on %.4f\n", r[0], r[1], r[2], r[3]); + check(!wrong, "and the plain row-major reading of the same bytes is not"); +} + +} // namespace + +int main(int argc, char** argv) { + const fs::path root = fs::temp_directory_path() / ("dflash-draft-test-" + std::to_string(::getpid())); + fs::remove_all(root); + test_synthetic(root); + fs::remove_all(root); + const char* real = (argc > 1 && argv[1][0] != '\0') ? argv[1] : std::getenv("ONEBIT_TEST_DFLASH_DRAFT"); + if (real != nullptr && *real != '\0') test_real(real); + else std::printf("SKIP the real package: pass its directory, or set ONEBIT_TEST_DFLASH_DRAFT\n"); + if (g_failures == 0) std::printf("dflash_draft: ok (%d checks)\n", g_checks); + return g_failures == 0 ? 0 : 1; +}