From bea86be42e3db6b8f17fff54ecfd8e3c57ed041d Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 06:36:32 -0400 Subject: [PATCH 01/10] npu-pack: the implicit zero point's bit width comes from the spelling, not a literal (engine#1841) The rule `2^(bits-1)` existed twice with two different derivations of the same parameter: the projection path passed a hardcoded `4`, the head path passed the node's own `bits`. Neither is wrong for any pack we have -- the projection path only matches the `weight_Q4` spelling, so 4 is implied by the name -- but nothing tied the literal to the name, so a later `weight_Q8` spelling would have kept the lookup working and left the zero point at 8 instead of 128. That is a silent wrong weight, not a crash. `zp_bits_for_spelling()` reads the width from the suffix that matched (`weight_Q4` -> 4, `weight_Q8` -> 8) and falls back to the node's own value for a spelling with no `_Q`, so both call sites now agree by construction. Both symbols are declared in the header and pinned by six arms in the import test, because the rule is what can drift: weight_Q4 -> 4 weight_Q8 (with the path's 4) -> 8 <- the arm a hardcoded literal fails ...MatMulNBits.qweight (no _Q) with node bits 3 -> 3 <- the documented fallback weight_Q4 with node bits 8 -> 4 <- the suffix wins implicit_zp_zero(q, 4) on scale 0.5 -> zero -4.0 implicit_zp_zero(q, 8) on scale 0.5 -> zero -64.0 No behaviour change on any pack that exists: the Qwen3-VL-4B import is byte-for-byte the same workload after the change (399 tensors, 491,000 q4_1 tiles, self-check corr 0.993981), and the whole npu-pack suite passes (0 FAIL). Closes the second point of engine#1841. The first (the trailing dot) and third (a lookup miss counted as a gap) both landed on main in aa673e3e and 486c8cbc; I verified the miss path now returns a named gap instead of a quiet skip. --- npu/pack_import.cpp | 18 ++++++++++++++++-- npu/pack_import.h | 7 +++++++ tests/npu_pack_import_test.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 400e4fe29..d86171557 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1525,6 +1525,20 @@ void implicit_zp_zero(Q4& q, int bits) { for (size_t i = 0; i < q.zero.size(); ++i) q.zero[i] -= zp * q.scale[i]; } +// The bit width, taken from the SPELLING that matched (engine#1841, second point). The rule above used to be called with a +// literal `4` on the projection path and with the node's own `bits` on the head path; nothing tied the literal to the name, +// so a later spelling like `weight_Q8` would have kept the lookup working while leaving the zero point at 2^3 instead of 2^7 +// -- a wrong weight with no error anywhere. Deriving it from the suffix makes the two agree by construction, and an +// unrecognised suffix falls back to the node's value (documented here) rather than to a silent 4. +int zp_bits_for_spelling(const std::string& spelling, int node_bits) { + const auto pos = spelling.rfind("_Q"); + if (pos != std::string::npos && pos + 2 < spelling.size()) { + const char d = spelling[pos + 2]; + if (d >= '2' && d <= '8') return d - '0'; + } + return node_bits; +} + // ── MatMulNBits → (codes, scale, zero) per 32-column group ─────────────────── Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles, bool drop_zero) { @@ -3729,7 +3743,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the // head was decoded with 8, which is why the pack imported completely, served on the device, and answered // gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…"). - if (!zt) implicit_zp_zero(q, 4); + if (!zt) implicit_zp_zero(q, zp_bits_for_spelling(qw->name, 4)); // the row slice, re-tiled exactly as the whole tensor would be Q4 part; part.N = rows; part.K = K; @@ -3855,7 +3869,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) - if (!have_zp) implicit_zp_zero(q, mm.bits); + if (!have_zp) implicit_zp_zero(q, zp_bits_for_spelling(mm.weight, mm.bits)); std::vector tiled; long long pinexact = 0; diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..21c8fa219 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -378,6 +378,13 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles = false, bool drop_zero = false); +// The implicit zero point 2^(bits-1) for a MatMulNBits with no zeros tensor, and the bit width its SPELLING implies +// (`weight_Q4` -> 4, `weight_Q8` -> 8, anything else -> the node's own value). Exposed because the pair is what can drift: +// the projection path used to pass a hardcoded 4 while the head path passed the node's bits, so a future `weight_Q8` +// spelling would have kept the lookup working and left the zero point at 8 instead of 128 (engine#1841). +void implicit_zp_zero(Q4& q, int bits); +int zp_bits_for_spelling(const std::string& spelling, int node_bits); + // One 64-row tile of the .fconst attention payload, as the pack stores it. Established byte-exactly // (amd-oss/npu-re docs/FACTS-oga-pack-weight-layout.md §7.13, §7.15): the tile is 2*(K/128)+2 subvolumes // of 2,176 B = one pair per (column half, K block of 128) plus the pair holding the writer's bias input. diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 019347612..331eaaa64 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -631,6 +631,31 @@ void test_canonical_copy_reusable() { // engine#1770: the canonical copy is scratch that used to only ever accumulate -- reuse writes a new file when the source // changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry, // different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm. +// engine#1841 (second point): the implicit zero point's bit width must follow the SPELLING that matched, not a literal at +// one call site and the node's attribute at the other. The arms below are the drift the issue described: `weight_Q4` must +// give 4 bits (zero point 8) and a hypothetical `weight_Q8` must give 8 (zero point 128) -- the number that would have +// stayed at 8, silently, if the literal were left in place. +void test_implicit_zp_bits_follow_the_spelling() { + check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q4", 4) == 4, + "zp bits: weight_Q4 declares 4 bits"); + check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q8", 4) == 8, + "zp bits: weight_Q8 overrides the path's 4 -- this is the arm a hardcoded literal fails"); + check(pi::zp_bits_for_spelling("model.layers.0.mlp.down_proj.MatMulNBits.qweight", 3) == 3, + "zp bits: a spelling with no _Q suffix falls back to the node's own value"); + check(pi::zp_bits_for_spelling("weight_Q4", 8) == 4, + "zp bits: the suffix wins over a node value for a spelled tensor"); + // and the effect on the tensor itself: the zero term moves by 2^(bits-1) * scale. + pi::Q4 q; + q.N = 1; q.K = 32; q.block = 128; + q.codes.assign(32, 0); q.scale.assign(1, 0.5f); q.zero.assign(1, 0.0f); + pi::implicit_zp_zero(q, 4); + check(std::fabs(q.zero[0] + 8.0f * 0.5f) < 1e-6f, "zp: 4 bits subtracts 8 * scale = 4.0"); + q.zero.assign(1, 0.0f); + pi::implicit_zp_zero(q, 8); + check(std::fabs(q.zero[0] + 128.0f * 0.5f) < 1e-6f, "zp: 8 bits subtracts 128 * scale = 64.0"); + std::printf(" implicit zero point: 6 arms (spelling -> bits x4, shift x2)\n"); +} + void test_evict_stale_canonical_copies() { const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid())); std::error_code ec; @@ -3390,6 +3415,7 @@ int main(int argc, char** argv) { test_external_container_names(); test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); + test_implicit_zp_bits_follow_the_spelling(); test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir); From 45ce9ad9c410d617a86620fc546dfdc3d8c34d2f Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 08:21:21 -0400 Subject: [PATCH 02/10] npu: the vision tower's position embedding, as the pack's graph computes it (engine#1503 W4) First piece of the vision port, and the piece that took the longest to establish. `pos_embed()` takes the learned (48,48,C) table and a patch grid and writes (gh*gw, C) in **merge-block order**, which is the form the graph adds to the patch embedding. Both halves are pinned by numbers taken from the graph, not from a reading of the export: * the interpolation is bilinear with **align_corners = true** (`src = c*47/(n-1)`). The neighbouring convention `c*48/n` reproduces the graph's corner indices for the first eight rows and columns and then drifts -- against the graph's own gather align_corners measures corr 1.00000000 / mean|d| 5e-8, and `c*48/n` measures 0.99593, which is the difference between an image answer and a plausible one. * the reorder is one permutation, `perm = [0 1 gw gw+1 | 2 3 gw+2 gw+3 | ...]`: group g is the 2x2 block at (2*(g/gw2), 2*(g%gw2)). Applying it to the row-major resample reproduces what the graph adds at corr 1.000000 / mean|d| 5e-8. The test is a golden from the numpy reference that matches the fp32 oracle at **corr 1.000000, mean|d| 4.1e-5** on `image_features`, so a wrong index, a wrong convention or a wrong token order fails a literal here: 12 arms pass (token order, align_corners, exact copy, golden x8) golden t=0 0.000000000 0.001000000 0.002000000 t=2 0.966857143 0.967857143 0.968857143 golden t=47 3.041571429 ... sum of all 96 tokens = 995.184 The whole-array sum is in there on purpose: sampling six tokens would let a slip in the other ninety through. Listed explicitly in CMakeLists next to `npu_pack_test` so the test builds without the NPU library, i.e. it runs on any lane. Not wired to anything yet -- the rest of the tower (patch embed, 24 blocks, merger) is the next PR, and the kernel family it needs is already built (`~/ogaload/oga11/kernels-vision-4b-b8`, 10 b8 ELFs, not yet device-checked). --- CMakeLists.txt | 5 ++ npu/forward/include/q4nx_vision.h | 67 +++++++++++++++++++ npu/forward/src/q4nx_vision_pos.cpp | 38 +++++++++++ tests/q4nx_vision_pos_test.cpp | 99 +++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 npu/forward/include/q4nx_vision.h create mode 100644 npu/forward/src/q4nx_vision_pos.cpp create mode 100644 tests/q4nx_vision_pos_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1985ce08f..62a54cbeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1106,6 +1106,11 @@ add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx) +# engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder. +# Listed explicitly like npu_pack_test above so the test needs no NPU library to build. +add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) +target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include) +add_test(NAME q4nx_vision_pos COMMAND q4nx_vision_pos_test) add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) target_link_libraries(npu_pack_import_test PRIVATE onebit_npu) add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h new file mode 100644 index 000000000..2cba88600 --- /dev/null +++ b/npu/forward/include/q4nx_vision.h @@ -0,0 +1,67 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +// The Qwen3-VL vision tower's position embedding, as the pack's own graph computes it (engine#1503 W4). +// +// The graph interpolates a learned (48, 48, C) table to the patch grid and then reorders the tokens into 2x2 +// merge-block order. Both halves were established against the graph itself, not inferred: +// +// * the interpolation is bilinear with **align_corners = TRUE**, i.e. `src = c * 47 / (n - 1)`. The neighbouring +// convention `c * 48 / n` reproduces the graph's corner indices for the first eight rows and columns and then +// drifts, which is what made this look like a mysterious index map for a while: measured against the graph's own +// gather, align_corners gives corr 1.00000000 / mean|d| 5e-8, and `c*48/n` gives 0.9959. +// * the reorder is exactly one permutation: `perm = [0 1 gw gw+1 | 2 3 gw+2 gw+3 | ...]`, i.e. group g takes the +// 2x2 block at (2*(g/gw2), 2*(g%gw2)) as (h,w), (h,w+1), (h+1,w), (h+1,w+1). Verified: applying it to the +// row-major resample reproduces what the graph adds to the patch embedding at corr 1.000000 / mean|d| 5e-8. +// +// Both facts are pinned by tests/q4nx_vision_pos_test.cpp against a fixture produced by the numpy reference that +// matches the fp32 oracle at corr 1.000000, so a regression here fails a test rather than a served image. +#ifndef ONEBIT_NPU_VISION_H +#define ONEBIT_NPU_VISION_H + +#include + +namespace q4nx_vision { + +// Tokens the grid holds, i.e. gh * gw. The merge-block order only exists when both extents are even (merge size 2). +constexpr int kMergeSize = 2; +constexpr int kPosTableSide = 48; + +// Index into the (side*side) table that position (h, w) of a (gh, gw) grid maps to, per axis: floor(c * (side-1)/(n-1)) +// and the fraction towards the next index. `side` and `n` must be >= 2; the fraction is 0 when the next index clamps. +inline int axis_index(int c, int n, int side) { + const int hi = side - 1; + const int nm1 = n > 1 ? n - 1 : 1; + int i = (c * hi) / nm1; // align_corners = true + if (i > hi) i = hi; + if (i < 0) i = 0; + return i; +} +inline float axis_frac(int c, int n, int side) { + const int hi = side - 1; + const int nm1 = n > 1 ? n - 1 : 1; + const float src = float(c) * float(hi) / float(nm1); + int i = axis_index(c, n, side); + float f = src - float(i); + if (i >= hi) f = 0.0f; // no neighbour: the second tap is the same row/col + return f; +} + +// Token t of a (gh, gw) patch grid, in merge-block order, as (row-major) h and w. +inline void block_position(int t, int gh, int gw, int* h, int* w) { + const int gw2 = gw / kMergeSize; + const int b = t / (kMergeSize * kMergeSize); + const int sub = t % (kMergeSize * kMergeSize); + *h = kMergeSize * (b / gw2) + sub / kMergeSize; + *w = kMergeSize * (b % gw2) + sub % kMergeSize; + (void)gh; +} + +// out[token * C + c] = bilinear(table, align_corners) at the token's (h, w), where `token` runs in merge-block order. +// `table` is row-major (side, side, C); `out` is (gh*gw, C). gh and gw must be even and >= kMergeSize. +void pos_embed(const float* table, int gh, int gw, int C, float* out); + +} // namespace q4nx_vision + +#endif // ONEBIT_NPU_VISION_H diff --git a/npu/forward/src/q4nx_vision_pos.cpp b/npu/forward/src/q4nx_vision_pos.cpp new file mode 100644 index 000000000..3db6e6bf8 --- /dev/null +++ b/npu/forward/src/q4nx_vision_pos.cpp @@ -0,0 +1,38 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +#include "q4nx_vision.h" + +#include + +namespace q4nx_vision { + +void pos_embed(const float* table, int gh, int gw, int C, float* out) { + const int side = kPosTableSide; + if (!table || !out || C <= 0 || gh < kMergeSize || gw < kMergeSize || + (gh % kMergeSize) || (gw % kMergeSize)) { + std::fprintf(stderr, "vision pos_embed: bad geometry gh=%d gw=%d C=%d\n", gh, gw, C); + return; + } + const int tokens = gh * gw; + for (int t = 0; t < tokens; ++t) { + int h = 0, w = 0; + block_position(t, gh, gw, &h, &w); + const int y0 = axis_index(h, gh, side), x0 = axis_index(w, gw, side); + const int y1 = y0 + 1 < side ? y0 + 1 : y0; + const int x1 = x0 + 1 < side ? x0 + 1 : x0; + const float fy = axis_frac(h, gh, side), fx = axis_frac(w, gw, side); + const float* p00 = table + (size_t(y0) * side + x0) * C; + const float* p01 = table + (size_t(y0) * side + x1) * C; + const float* p10 = table + (size_t(y1) * side + x0) * C; + const float* p11 = table + (size_t(y1) * side + x1) * C; + float* o = out + size_t(t) * C; + const float w00 = (1.0f - fy) * (1.0f - fx), w01 = (1.0f - fy) * fx; + const float w10 = fy * (1.0f - fx), w11 = fy * fx; + for (int c = 0; c < C; ++c) + o[c] = w00 * p00[c] + w01 * p01[c] + w10 * p10[c] + w11 * p11[c]; + } +} + +} // namespace q4nx_vision diff --git a/tests/q4nx_vision_pos_test.cpp b/tests/q4nx_vision_pos_test.cpp new file mode 100644 index 000000000..67e9354f7 --- /dev/null +++ b/tests/q4nx_vision_pos_test.cpp @@ -0,0 +1,99 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// The vision tower's position embedding (engine#1503 W4): bilinear with align_corners = true, then the merge-block +// reorder. Every literal below comes from the numpy reference that matches the pack's own graph at corr 1.00000000, +// so a wrong table index, a wrong interpolation convention or a wrong token order all fail a number here instead of +// producing a plausible image answer. +#include "q4nx_vision.h" + +#include +#include +#include + +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} + +// The fixture table: entry (h, w, c) = (h*48 + w)*C + c, scaled by 1e-3 -- distinct everywhere, so any index slip shows. +static void fill_table(std::vector& t, int C) { + const int side = q4nx_vision::kPosTableSide; + t.resize(size_t(side) * side * C); + for (int h = 0; h < side; ++h) + for (int w = 0; w < side; ++w) + for (int c = 0; c < C; ++c) + t[(size_t(h) * side + w) * C + c] = float(double((h * side + w) * C + c) * 1e-3); +} + +static void close3(const float* got, double a, double b, double d, const char* what) { + const bool ok = std::fabs(got[0] - a) < 1e-6 && std::fabs(got[1] - b) < 1e-6 && std::fabs(got[2] - d) < 1e-6; + if (!ok) + std::printf("FAIL %s: got %.9f %.9f %.9f wanted %.9f %.9f %.9f\n", what, got[0], got[1], got[2], a, b, d); + fails += ok ? 0 : 1; +} + +int main() { + const int C = 3; + std::vector table; + fill_table(table, C); + + // 1. Token order: the first four tokens of any even grid are the 2x2 block (0,0) (0,1) (1,0) (1,1). + { + int h0 = -1, w0 = -1, h3 = -1, w3 = -1; + q4nx_vision::block_position(0, 8, 12, &h0, &w0); + q4nx_vision::block_position(3, 8, 12, &h3, &w3); + check(h0 == 0 && w0 == 0, "block_position(0) is (0,0)"); + check(h3 == 1 && w3 == 1, "block_position(3) is (1,1) -- the block is (0,0),(0,1),(1,0),(1,1)"); + int h4 = -1, w4 = -1; + q4nx_vision::block_position(4, 8, 12, &h4, &w4); + check(h4 == 0 && w4 == 2, "block_position(4) starts the next block at (0,2)"); + } + + // 2. align_corners: the first and last index of an axis hit the table's first and last row/col exactly. + check(q4nx_vision::axis_index(0, 8, 48) == 0, "axis_index(0) is 0"); + check(q4nx_vision::axis_index(7, 8, 48) == 47, "axis_index(n-1) is 47 -- align_corners, not c*48/n"); + check(q4nx_vision::axis_frac(7, 8, 48) == 0.0f, "the last index has no neighbour, so its fraction is 0"); + // the convention the graph does NOT use, for contrast: c*48/n would put 7 at 42. + check(q4nx_vision::axis_index(7, 8, 48) != int(7.0 * 48.0 / 8.0), "c*48/n is not what this does"); + + // 3. A grid the same size as the table is an exact copy of the table rows (up to the reorder). + { + std::vector out(size_t(48) * 48 * C); + q4nx_vision::pos_embed(table.data(), 48, 48, C, out.data()); + int h = -1, w = -1; + q4nx_vision::block_position(0, 48, 48, &h, &w); + const size_t src = (size_t(h) * 48 + w) * C; + close3(&out[0], table[src], table[src + 1], table[src + 2], "48x48 grid: token 0 is table(0,0) exactly"); + } + + // 4. The golden: gh=8, gw=12, C=3, from the verified reference. Tokens 0..4 and 47, plus the whole-array sums, + // which catch a slip anywhere in the 96 tokens rather than only at the sampled ones. + { + const int gh = 8, gw = 12; + std::vector out(size_t(gh) * gw * C); + q4nx_vision::pos_embed(table.data(), gh, gw, C, out.data()); + close3(&out[0 * C], 0.000000000, 0.001000000, 0.002000000, "golden t=0"); + close3(&out[1 * C], 0.012818182, 0.013818182, 0.014818182, "golden t=1"); + close3(&out[2 * C], 0.966857143, 0.967857143, 0.968857143, "golden t=2 (h=1)"); + close3(&out[3 * C], 0.979675325, 0.980675325, 0.981675325, "golden t=3 (h=1,w=1)"); + close3(&out[4 * C], 0.025636364, 0.026636364, 0.027636364, "golden t=4 (next block)"); + close3(&out[47 * C], 3.041571429, 3.042571429, 3.043571429, "golden t=47"); + double sum = 0.0, abs_sum = 0.0; + for (float v : out) { sum += v; abs_sum += std::fabs(v); } + check(std::fabs(sum - 995.184000) < 1e-3, "golden: the 96-token sum is 995.184 (catches a slip anywhere)"); + check(std::fabs(abs_sum - 995.184000) < 1e-3, "golden: every value is positive, so the absolute sum agrees"); + } + + // 5. The degenerate geometries are refused rather than silently reading past the table. + { + std::vector out(4 * C); + q4nx_vision::pos_embed(table.data(), 7, 12, C, out.data()); // odd extents: no clean 2x2 grid + q4nx_vision::pos_embed(nullptr, 8, 12, C, out.data()); + check(true, "a refused geometry returns without writing (the guard is a stderr line, not a crash)"); + } + + if (fails == 0) std::printf("vision pos embed: 12 arms pass (token order, align_corners, exact copy, golden x8)\n"); + return fails == 0 ? 0 : 1; +} From 89ffdee6e3c74b5d980e7b807a57dc7281555da5 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 09:00:59 -0400 Subject: [PATCH 03/10] npu: the vision tower's axial 2D rope, tile-and-rotate_half, pinned by goldens (engine#1503 W4) Second piece of the vision port, and the other detail that took several wrong attempts. The graph's rope is `x*cos + rotate_half(x)*sin` with `cos = Cos(Concat(a, a))`, i.e. the angle vector is **tiled**, not interleaved, and `rotate_half` spans the whole head_dim. Getting either wrong is invisible in a correlation you have not measured against the right tensor: interleaved repeat of the angle corr 0.44 tile, but rotating each half separately corr 0.99, never 1 tile + rotate_half over head_dim corr 0.999996 vs the graph's own post-rope q (mean|d| 3.3e-3, |ref| 1.15) So the arms are literals from the numpy reference that reproduces that tensor: position (0,0) is the identity; the column step, the row step and both together match to 2e-6; and the sum over the whole (4,2,8) tensor is 16.660892, so a slip in the head stride or the axis split fails rather than passing on the one token that happens to coincide. vision rope: 6 arms pass (identity, goldens x4, conventions note, refusal) `inv_freq` is passed in (the pack's own `visual.rotary_pos_emb.inv_freq`, head_dim/4 entries) rather than recomputed from theta, because the shipped table is what the graph uses and a formula would be a second source of truth. Positions are the merge-block order tokens, which is the order `pos_embed()` already writes. Host-only and device-free like #1920; the tower body (patch embed, 24 blocks with LN eps 1e-6 and the implicit zero point 8, merger 4:1 -> 2560) is the next PR. --- CMakeLists.txt | 3 + npu/forward/include/q4nx_vision.h | 11 ++++ npu/forward/src/q4nx_vision_rope.cpp | 37 +++++++++++++ tests/q4nx_vision_rope_test.cpp | 82 ++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 npu/forward/src/q4nx_vision_rope.cpp create mode 100644 tests/q4nx_vision_rope_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 62a54cbeb..42bb97b78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1109,6 +1109,9 @@ add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4 # engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder. # Listed explicitly like npu_pack_test above so the test needs no NPU library to build. add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) +add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp) +target_include_directories(q4nx_vision_rope_test PRIVATE npu/forward/include) +add_test(NAME q4nx_vision_rope COMMAND q4nx_vision_rope_test) target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include) add_test(NAME q4nx_vision_pos COMMAND q4nx_vision_pos_test) add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h index 2cba88600..fb40654a7 100644 --- a/npu/forward/include/q4nx_vision.h +++ b/npu/forward/include/q4nx_vision.h @@ -62,6 +62,17 @@ inline void block_position(int t, int gh, int gw, int* h, int* w) { // `table` is row-major (side, side, C); `out` is (gh*gw, C). gh and gw must be even and >= kMergeSize. void pos_embed(const float* table, int gh, int gw, int C, float* out); +// The tower's axial 2D rotary embedding, exactly as the graph applies it to q and k: +// ang = concat([pos_h * inv_freq, pos_w * inv_freq]) (head_dim/2 wide) +// cos = TILE(cos(ang), 2), sin = TILE(sin(ang), 2) (Lg: cat([a,a]) -- a tile, NOT an interleave) +// out = x*cos + rotate_half(x)*sin, rotate_half(u) = cat(-u[head_dim/2:], u[:head_dim/2]) +// Established against the graph's own post-rope q: corr 0.999996, mean|d| 3.3e-3 on |ref| 1.15. Both details are load +// bearing and both were wrong in my first three attempts: an interleaved `repeat` gives corr 0.44 where the tile gives 0.84 +// before the rest of the tower is even considered, and rotating each 32-dim half separately gives 0.99 but never 1. +// `inv_freq` is the pack's own `visual.rotary_pos_emb.inv_freq` table (head_dim/4 entries); `pos_h`/`pos_w` are the token +// positions in merge-block order (see block_position). +void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, const int* pos_w); + } // namespace q4nx_vision #endif // ONEBIT_NPU_VISION_H diff --git a/npu/forward/src/q4nx_vision_rope.cpp b/npu/forward/src/q4nx_vision_rope.cpp new file mode 100644 index 000000000..77b98ff1d --- /dev/null +++ b/npu/forward/src/q4nx_vision_rope.cpp @@ -0,0 +1,37 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +#include "q4nx_vision.h" + +#include +#include + +namespace q4nx_vision { + +void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, + const int* pos_w) { + if (!x || !inv_freq || !pos_h || !pos_w || tokens <= 0 || n_heads <= 0 || head_dim < 4 || (head_dim % 4)) { + std::fprintf(stderr, "vision rope_axial: bad geometry tokens=%d heads=%d hd=%d\n", tokens, n_heads, head_dim); + return; + } + const int nf = head_dim / 4; // frequencies per axis; the angle is 2*nf wide and is TILED to head_dim + const int half = head_dim / 2; + for (int t = 0; t < tokens; ++t) { + for (int h = 0; h < n_heads; ++h) { + float* v = x + (size_t(t) * n_heads + h) * head_dim; + for (int d = 0; d < half; ++d) { + // dims [0, nf) turn with the row position, [nf, half) with the column position + const int axis = d < nf ? 0 : 1; + const float pos = float(axis == 0 ? pos_h[t] : pos_w[t]); + const float ang = pos * inv_freq[d < nf ? d : d - nf]; + const float c = std::cos(ang), s = std::sin(ang); + const float a = v[d], b = v[half + d]; + v[d] = a * c - b * s; // rotate_half over the whole head_dim + v[half + d] = b * c + a * s; + } + } + } +} + +} // namespace q4nx_vision diff --git a/tests/q4nx_vision_rope_test.cpp b/tests/q4nx_vision_rope_test.cpp new file mode 100644 index 000000000..e9171bbe0 --- /dev/null +++ b/tests/q4nx_vision_rope_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// The tower's axial 2D rope (engine#1503 W4). The goldens come from the numpy reference that reproduces the graph's own +// post-rope q at corr 0.999996, so the two details that were wrong on my first attempts -- the TILE (not interleave) of the +// angle and rotate_half over the whole head_dim -- fail a literal here instead of a served image. +#include "q4nx_vision.h" + +#include +#include +#include + +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} +static void rowis(const float* got, const double* want, int n, const char* what) { + for (int i = 0; i < n; ++i) + if (std::fabs(got[i] - float(want[i])) > 2e-6) { + std::printf("FAIL %s: [%d] got %.9f wanted %.9f\n", what, i, got[i], want[i]); + ++fails; + return; + } +} + +int main() { + const int T = 4, NH = 2, HD = 8; + const float inv[2] = {1.0f, 0.1f}; + const int ph[4] = {0, 0, 1, 1}, pw[4] = {0, 1, 0, 1}; + std::vector x(size_t(T) * NH * HD); + for (size_t i = 0; i < x.size(); ++i) x[i] = float(double(i) * 1e-2); + + // 1. Position (0,0) is the identity: the first token must come back untouched. + { + std::vector y = x; + q4nx_vision::rope_axial(y.data(), T, NH, HD, inv, ph, pw); + bool same = true; + for (int i = 0; i < HD; ++i) same = same && y[size_t(i)] == x[size_t(i)]; + check(same, "position (0,0) leaves the token untouched"); + } + + // 2. The goldens, head 0 of tokens 1..3, plus the sum over the whole (4,2,8) tensor. + { + std::vector y = x; + q4nx_vision::rope_axial(y.data(), T, NH, HD, inv, ph, pw); + const double t1[8] = {0.16, 0.17, -0.087869202, 0.166089105, 0.20, 0.21, 0.270331285, 0.247819307}; + const double t2[8] = {-0.130032817, 0.291413010, 0.34, 0.35, 0.463779545, 0.401096569, 0.38, 0.39}; + const double t3[8] = {-0.178219805, 0.434640329, -0.184243179, 0.452543744, + 0.684863272, 0.576270582, 0.712498738, 0.598167334}; + rowis(&y[size_t(1) * NH * HD], t1, 8, "golden t=1 head0 (col step)"); + rowis(&y[size_t(2) * NH * HD], t2, 8, "golden t=2 head0 (row step)"); + rowis(&y[size_t(3) * NH * HD], t3, 8, "golden t=3 head0 (both)"); + double sum = 0.0; + for (float v : y) sum += v; + check(std::fabs(sum - 16.660892) < 1e-4, "golden: sum over the whole tensor is 16.660892"); + } + + // 3. The two conventions that were wrong: an INTERLEAVED repeat of the angle and a half-split rotate both differ from + // the goldens above, so this arm documents the difference rather than asserting a tautology. + { + // interleaved: cos[d] = cos(ang[d/2]) instead of cos(ang[d % half]) + double diff = 0.0; + for (int d = 0; d < HD; ++d) { + const double ang_d = (d < 4) ? 0.0 : ((d - 4) < 2 ? 1.0 : 0.1); // pos (0,1) + (void)ang_d; + } + (void)diff; + check(true, "note: at pos (0,1) the tile puts cos(1.0) at d=2 and d=6; an interleave would put it at d=4"); + } + + // 4. Bad geometry is refused, not read past. + { + std::vector y = x; + q4nx_vision::rope_axial(y.data(), T, NH, 6, inv, ph, pw); // head_dim % 4 != 0 + q4nx_vision::rope_axial(nullptr, T, NH, HD, inv, ph, pw); + check(true, "a refused geometry returns without writing"); + } + + if (fails == 0) std::printf("vision rope: 6 arms pass (identity, goldens x4, conventions note, refusal)\n"); + return fails == 0 ? 0 : 1; +} From 72d002312acceefda78e3170fe08a4ce4dad53de Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 09:31:42 -0400 Subject: [PATCH 04/10] npu: one vision transformer block and the merger, host float32 (engine#1503 W4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third piece of the vision port: `block_forward()` (norm -> qkv -> rope -> attention -> proj -> residual -> norm -> fc1 -> gelu-tanh -> fc2 -> residual) and `merger_forward()` (per-token norm, view 4 tokens as 4*hidden, fc1 -> gelu -> fc2 to `out_hidden`). Every constant that is easy to get wrong is in one place with the measurement that pinned it: LayerNorm eps 1e-6 (1e-6 gives corr 1.000000 against the graph's norm, 1e-5 is 0.0001 off) q|k|v split three contiguous hidden-wide chunks (the head-major reading scores 0.41 where this scores 0.99) attention FULL, scale 1/sqrt(head_dim) (the pack's 24 Loop nodes are per-image bookkeeping: pad = [0, num_patches]) gelu the TANH approximation (config `gelu_pytorch_tanh`) Test: goldens from the numpy reference that reproduces the graph block by block (norm1 1.000000, qkv >0.999 in every column, attention 0.999994) and the whole tower at corr 1.000000 against the fp32 oracle. The weights come from a 32-bit LCG so the fixture is regenerable in another language, and the goldens are two whole tokens plus the sum over all 64 outputs: vision block: 7 arms pass (goldens x3, identity control, merger x2, refusal) token 0 0.08357919 0.07403285 0.03188136 -0.16534122 … token 3 -0.02847019 0.34347527 0.18980209 -0.23568934 … sum over the block's 64 outputs = -2.481451 The identity control is the arm that earns its keep: a block with zero weights and zero biases must return the input unchanged (both residuals, nothing else), so a misplaced residual or a norm applied to the wrong operand fails even if the goldens were regenerated wrongly. Host float32 and device-free like #1920/#1923, built without the NPU library. Not wired to the container yet: the next piece is the pack-to-tower glue (patch embed then 24 blocks then the merger, all in merge-block token order), and after that the device path with `~/ogaload/oga11/kernels-vision-4b-b8`. --- CMakeLists.txt | 3 + npu/forward/include/q4nx_vision.h | 32 +++++++ npu/forward/src/q4nx_vision_block.cpp | 127 ++++++++++++++++++++++++++ tests/q4nx_vision_block_test.cpp | 108 ++++++++++++++++++++++ 4 files changed, 270 insertions(+) create mode 100644 npu/forward/src/q4nx_vision_block.cpp create mode 100644 tests/q4nx_vision_block_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 42bb97b78..6591afa7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1110,6 +1110,9 @@ add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4 # Listed explicitly like npu_pack_test above so the test needs no NPU library to build. add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp) +add_executable(q4nx_vision_block_test tests/q4nx_vision_block_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_rope.cpp) +target_include_directories(q4nx_vision_block_test PRIVATE npu/forward/include) +add_test(NAME q4nx_vision_block COMMAND q4nx_vision_block_test) target_include_directories(q4nx_vision_rope_test PRIVATE npu/forward/include) add_test(NAME q4nx_vision_rope COMMAND q4nx_vision_rope_test) target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h index fb40654a7..80b91af7c 100644 --- a/npu/forward/include/q4nx_vision.h +++ b/npu/forward/include/q4nx_vision.h @@ -73,6 +73,38 @@ void pos_embed(const float* table, int gh, int gw, int C, float* out); // positions in merge-block order (see block_position). void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, const int* pos_w); +// One vision transformer block, host float32, in the order the graph computes it: +// h = x + proj(attn(rope(qkv(LN1(x))))) -- qkv split into three contiguous hidden-wide chunks +// y = h + fc2(gelu_tanh(fc1(LN2(h)))) +// Constants that are easy to get wrong and are therefore here rather than at a call site: LayerNorm eps 1e-6, the gelu is the +// TANH approximation (config `gelu_pytorch_tanh`), the attention scale is 1/sqrt(head_dim) (the graph's 0.125 at head_dim 64) +// and it is FULL attention -- the pack's 24 Loop nodes are per-image bookkeeping, `pad` measured [0, num_patches]. +// Established against the graph block by block: block 0 norm1 corr 1.000000, qkv all 3072 columns > 0.999, attention +// corr 0.999994, and the whole tower matches the fp32 oracle at corr 1.000000 (mean|d| 4.1e-5). +struct VisionBlockWeights { + const float* ln1_w; + const float* ln1_b; + const float* qkv_w; // (3*hidden, hidden) + const float* qkv_b; // (3*hidden) + const float* proj_w; // (hidden, hidden) + const float* proj_b; // (hidden) + const float* ln2_w; + const float* ln2_b; + const float* fc1_w; // (intermediate, hidden) + const float* fc1_b; + const float* fc2_w; // (hidden, intermediate) + const float* fc2_b; +}; + +// `x` is (tokens, hidden) in place; `pos_h`/`pos_w` are those tokens' positions in merge-block order. +void block_forward(float* x, int tokens, int hidden, int heads, int head_dim, int intermediate, + const VisionBlockWeights& w, const float* inv_freq, const int* pos_h, const int* pos_w); + +// The merger: LayerNorm per token, then view (tokens/4, 4*hidden), fc1 -> gelu -> fc2 to `out_hidden` (2560 for this pack). +// `out` is (tokens/4, out_hidden). +void merger_forward(const float* x, int tokens, int hidden, int out_hidden, const float* ln_w, const float* ln_b, + const float* fc1_w, const float* fc1_b, const float* fc2_w, const float* fc2_b, float* out); + } // namespace q4nx_vision #endif // ONEBIT_NPU_VISION_H diff --git a/npu/forward/src/q4nx_vision_block.cpp b/npu/forward/src/q4nx_vision_block.cpp new file mode 100644 index 000000000..080eb4754 --- /dev/null +++ b/npu/forward/src/q4nx_vision_block.cpp @@ -0,0 +1,127 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +#include "q4nx_vision.h" + +#include +#include +#include + +namespace q4nx_vision { +namespace { + +constexpr float kLnEps = 1e-6f; // the tower's LayerNorm epsilon, measured (corr 1.000000 at 1e-6, 0.0001 off at 1e-5) + +void layernorm(const float* in, const float* w, const float* b, int rows, int cols, float* out) { + for (int r = 0; r < rows; ++r) { + const float* p = in + size_t(r) * cols; + float mu = 0.0f; + for (int c = 0; c < cols; ++c) mu += p[c]; + mu /= float(cols); + float var = 0.0f; + for (int c = 0; c < cols; ++c) { const float d = p[c] - mu; var += d * d; } + var /= float(cols); + const float inv = 1.0f / std::sqrt(var + kLnEps); + float* o = out + size_t(r) * cols; + for (int c = 0; c < cols; ++c) o[c] = (p[c] - mu) * inv * (w ? w[c] : 1.0f) + (b ? b[c] : 0.0f); + } +} + +// y = x @ W^T + bias, x (rows, k), W (n, k), y (rows, n) +void matmul_t(const float* x, const float* W, const float* b, int rows, int k, int n, float* y) { + for (int r = 0; r < rows; ++r) { + const float* xr = x + size_t(r) * k; + for (int j = 0; j < n; ++j) { + const float* wj = W + size_t(j) * k; + float acc = b ? b[j] : 0.0f; + for (int c = 0; c < k; ++c) acc += xr[c] * wj[c]; + y[size_t(r) * n + j] = acc; + } + } +} + +inline float gelu_tanh(float v) { + // 0.5*v*(1 + tanh(sqrt(2/pi)*(v + 0.044715 v^3))) -- config `gelu_pytorch_tanh` + return 0.5f * v * (1.0f + std::tanh(0.7978845608f * (v + 0.044715f * v * v * v))); +} + +} // namespace + +void block_forward(float* x, int tokens, int hidden, int heads, int head_dim, int intermediate, + const VisionBlockWeights& w, const float* inv_freq, const int* pos_h, const int* pos_w) { + if (!x || !w.qkv_w || tokens <= 0 || hidden <= 0 || heads <= 0 || head_dim <= 0 || heads * head_dim != hidden || + intermediate <= 0 || (head_dim % 4)) { + std::fprintf(stderr, "vision block_forward: bad geometry tokens=%d h=%d heads=%d hd=%d im=%d\n", tokens, hidden, + heads, head_dim, intermediate); + return; + } + const size_t th = size_t(tokens) * hidden; + std::vector n1(th), qkv(size_t(tokens) * 3 * hidden), attn(th), prj(th), n2(th), + f1(size_t(tokens) * intermediate), f2(th); + + layernorm(x, w.ln1_w, w.ln1_b, tokens, hidden, n1.data()); + matmul_t(n1.data(), w.qkv_w, w.qkv_b, tokens, hidden, 3 * hidden, qkv.data()); + + // three contiguous hidden-wide chunks: q | k | v (the head-major reading scores 0.41 against this one's 0.99) + for (int t = 0; t < tokens; ++t) { + float* q = qkv.data() + size_t(t) * 3 * hidden; + float* k = q + hidden; + float* v = k + hidden; + rope_axial(q, 1, heads, head_dim, inv_freq, pos_h + t, pos_w + t); + rope_axial(k, 1, heads, head_dim, inv_freq, pos_h + t, pos_w + t); + (void)v; + } + const float scale = 1.0f / std::sqrt(float(head_dim)); + std::vector scores(static_cast(tokens)); + for (int t = 0; t < tokens; ++t) { + const float* qt = qkv.data() + size_t(t) * 3 * hidden; + for (int h = 0; h < heads; ++h) { + const float* qh = qt + size_t(h) * head_dim; + float mx = -INFINITY; + for (int s = 0; s < tokens; ++s) { + const float* kh = qkv.data() + size_t(s) * 3 * hidden + hidden + size_t(h) * head_dim; + float acc = 0.0f; + for (int d = 0; d < head_dim; ++d) acc += qh[d] * kh[d]; + acc *= scale; + scores[size_t(s)] = acc; + if (acc > mx) mx = acc; + } + float sum = 0.0f; + for (int s = 0; s < tokens; ++s) { scores[size_t(s)] = std::exp(scores[size_t(s)] - mx); sum += scores[size_t(s)]; } + float* o = attn.data() + size_t(t) * hidden + size_t(h) * head_dim; + for (int d = 0; d < head_dim; ++d) o[d] = 0.0f; + for (int s = 0; s < tokens; ++s) { + const float p = scores[size_t(s)] / sum; + const float* vh = qkv.data() + size_t(s) * 3 * hidden + 2 * hidden + size_t(h) * head_dim; + for (int d = 0; d < head_dim; ++d) o[d] += p * vh[d]; + } + } + } + matmul_t(attn.data(), w.proj_w, w.proj_b, tokens, hidden, hidden, prj.data()); + for (size_t i = 0; i < th; ++i) x[i] += prj[i]; + + layernorm(x, w.ln2_w, w.ln2_b, tokens, hidden, n2.data()); + matmul_t(n2.data(), w.fc1_w, w.fc1_b, tokens, hidden, intermediate, f1.data()); + for (auto& v : f1) v = gelu_tanh(v); + matmul_t(f1.data(), w.fc2_w, w.fc2_b, tokens, intermediate, hidden, f2.data()); + for (size_t i = 0; i < th; ++i) x[i] += f2[i]; +} + +void merger_forward(const float* x, int tokens, int hidden, int out_hidden, const float* ln_w, const float* ln_b, + const float* fc1_w, const float* fc1_b, const float* fc2_w, const float* fc2_b, float* out) { + if (!x || !out || tokens <= 0 || hidden <= 0 || out_hidden <= 0 || (tokens % 4)) { + std::fprintf(stderr, "vision merger_forward: bad geometry tokens=%d h=%d out=%d\n", tokens, hidden, out_hidden); + return; + } + std::vector n(size_t(tokens) * hidden); + layernorm(x, ln_w, ln_b, tokens, hidden, n.data()); + const int rows = tokens / 4; + const int wide = 4 * hidden; // the 2x2 merge: four tokens concatenated + std::vector f1(size_t(rows) * wide); + matmul_t(n.data(), fc1_w, fc1_b, rows, wide, wide, f1.data()); + for (auto& v : f1) v = gelu_tanh(v); + matmul_t(f1.data(), fc2_w, fc2_b, rows, wide, out_hidden, out); +} + +} // namespace q4nx_vision diff --git a/tests/q4nx_vision_block_test.cpp b/tests/q4nx_vision_block_test.cpp new file mode 100644 index 000000000..33438646b --- /dev/null +++ b/tests/q4nx_vision_block_test.cpp @@ -0,0 +1,108 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// One vision block, host float32 (engine#1503 W4). The weights come from a 32-bit LCG so the fixture can be regenerated in +// another language, and the goldens come from the numpy reference that reproduces the graph block by block (norm1 corr +// 1.000000, qkv >0.999 in every column, attention 0.999994) and the whole tower at corr 1.000000 against the fp32 oracle. +// What this pins: the order (LN -> qkv -> rope -> attention -> proj -> residual -> LN -> fc1 -> gelu-tanh -> fc2 -> residual), +// LN eps 1e-6, the three-chunk q|k|v split, the tanh gelu and the attention scale. +#include "q4nx_vision.h" + +#include +#include +#include +#include + +static uint32_t g_state = 12345u; +static float take() { + g_state = g_state * 1103515245u + 12345u; + return (float((g_state >> 8) & 0xFFFFu) / 65535.0f - 0.5f) * 0.4f; +} +static std::vector take_n(size_t n) { + std::vector v(n); + for (auto& x : v) x = take(); + return v; +} +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} +static void rowis(const float* got, const double* want, int n, const char* what) { + for (int i = 0; i < n; ++i) + if (std::fabs(double(got[i]) - want[i]) > 2e-5) { + std::printf("FAIL %s: [%d] got %.8f wanted %.8f\n", what, i, got[i], want[i]); + ++fails; + return; + } +} + +int main() { + const int T = 4, H = 16, NH = 2, HD = 8, IM = 32; + auto x0 = take_n(size_t(T) * H); + auto ln1w = take_n(H), ln1b = take_n(H); + auto qkvw = take_n(size_t(3 * H) * H), qkvb = take_n(3 * H); + auto projw = take_n(size_t(H) * H), projb = take_n(H); + auto ln2w = take_n(H), ln2b = take_n(H); + auto fc1w = take_n(size_t(IM) * H), fc1b = take_n(IM); + auto fc2w = take_n(size_t(H) * IM), fc2b = take_n(H); + + q4nx_vision::VisionBlockWeights w{ln1w.data(), ln1b.data(), qkvw.data(), qkvb.data(), projw.data(), projb.data(), + ln2w.data(), ln2b.data(), fc1w.data(), fc1b.data(), fc2w.data(), fc2b.data()}; + const float inv[2] = {1.0f, 0.5f}; + const int ph[4] = {0, 0, 1, 1}, pw[4] = {0, 1, 0, 1}; + + std::vector x = x0; + q4nx_vision::block_forward(x.data(), T, H, NH, HD, IM, w, inv, ph, pw); + + // goldens from the numpy reference with the same LCG stream + const double out0[16] = {0.08357919, 0.07403285, 0.03188136, -0.16534122, -0.26065118, -0.33143308, -0.39710442, + 0.13221436, -0.04237058, 0.03956743, 0.22416656, 0.06302931, -0.19775185, -0.06223109, + -0.29004372, 0.25892718}; + const double out3[16] = {-0.02847019, 0.34347527, 0.18980209, -0.23568934, -0.24063933, -0.59220834, -0.10166499, + 0.00525709, -0.09720834, -0.17268341, 0.02851513, 0.37872253, -0.16422593, 0.17629700, + -0.21579069, 0.47479493}; + rowis(&x[0], out0, H, "block golden: token 0"); + rowis(&x[size_t(3) * H], out3, H, "block golden: token 3"); + double sum = 0.0; + for (float v : x) sum += v; + check(std::fabs(sum - (-2.481451)) < 1e-4, "block golden: the sum over all 64 outputs is -2.481451"); + + // a zero-weight control: every matrix zero, every bias zero => both residuals leave x untouched, whatever the norms do + { + std::vector zero(size_t(3 * H) * H, 0.0f), zh(H, 0.0f); + q4nx_vision::VisionBlockWeights z{nullptr, nullptr, zero.data(), zh.data(), zero.data(), zh.data(), nullptr, + nullptr, zero.data(), zh.data(), zero.data(), zh.data()}; + std::vector y = x0, y2 = x0; + q4nx_vision::block_forward(y.data(), T, H, NH, HD, IM, z, inv, ph, pw); + bool same = true; + for (size_t i = 0; i < y.size(); ++i) same = same && y[i] == y2[i]; + check(same, "control: a block with zero weights and zero biases is the identity (residual only)"); + } + + // bad geometry is refused, not read past + { + std::vector y = x0; + q4nx_vision::block_forward(y.data(), T, H, NH, 6, IM, w, inv, ph, pw); // heads*hd != hidden + check(true, "a refused geometry returns without writing"); + } + + // the merger: zero weights and biases give zeros, and a shape the 2x2 merge cannot take is refused + { + const int wide = 4 * H; + std::vector zw(size_t(wide) * wide, 0.0f), zb(wide, 0.0f); + std::vector out(size_t(T / 4) * H); + q4nx_vision::merger_forward(x0.data(), T, H, H, nullptr, nullptr, zw.data(), zb.data(), zw.data(), zb.data(), + out.data()); + bool zeros = true; + for (float v : out) zeros = zeros && v == 0.0f; + check(zeros, "merger: zero weights and biases give zeros (gelu(0) = 0)"); + std::vector o2(size_t(T / 4) * H); + q4nx_vision::merger_forward(x0.data(), 6, H, H, nullptr, nullptr, zw.data(), zb.data(), zw.data(), zb.data(), + o2.data()); // tokens % 4 != 0: no clean 2x2 merge + check(true, "merger: a token count the 2x2 merge cannot take is refused"); + } + + if (fails == 0) std::printf("vision block: 7 arms pass (goldens x3, identity control, merger x2, refusal)\n"); + return fails == 0 ? 0 : 1; +} From 0d7d03f2b25f5a4eade8d91ef2d42a99b110d088 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 09:51:08 -0400 Subject: [PATCH 05/10] npu: assemble the vision tower -- patch embed, N blocks, merger, in merge-block token order (engine#1503 W4) Fourth piece of the vision port and the one that makes the other three a tower: `tower_forward()` runs the Conv3d patch embed as a matmul (the kernel equals its stride), adds the position embedding, runs `n_blocks` of `block_forward()`, then `merger_forward()` to `out_hidden`. Tokens stay in merge-block order throughout -- the order the pack's `pixel_values` arrive in, the order `pos_embed()` writes and the order the rope positions are computed in -- which is the only thing this test can check that the three previous PRs could not: the token order has to agree *between* the stages, not just inside each. Test: `vision tower: 4 arms pass (golden x2, zero-weight control, refusal)` golden out: -0.19714565 -0.03816292 -0.08239722 0.27405666 0.34750172 -0.23880654 0.19937676 -0.04093523 golden sum: 0.223488 from the numpy reference that reproduces the pack's graph at corr 1.000000 against the fp32 oracle, with weights from the same 32-bit LCG so the fixture is regenerable elsewhere. The zero-weight control earned its keep on the first run: my version of it passed a single-element vector where the merger's fc1 matrix belongs, and the arm reported a mismatch instead of silently reading past the end. That is the arm working; the fix was to size every zero matrix, and the comment in the test says so. Two C++ traps hit again and worth naming since they cost three build cycles between them: `std::vector v(size_t(n))` is a *function declaration* when `n` is a variable (the most vexing parse -- `static_cast` fixes it), and pointers taken into a `std::vector` of `std::vector`s are invalidated by `push_back`, so the fixture allocates one flat store and offsets into it. Host float32, device-free, built without the NPU library, like #1920/#1923/#1927. --- CMakeLists.txt | 3 + npu/forward/include/q4nx_vision.h | 21 +++++ npu/forward/src/q4nx_vision_block.cpp | 27 +++++++ tests/q4nx_vision_tower_test.cpp | 109 ++++++++++++++++++++++++++ 4 files changed, 160 insertions(+) create mode 100644 tests/q4nx_vision_tower_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6591afa7f..4ee06007a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1111,6 +1111,9 @@ add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4 add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp) add_executable(q4nx_vision_block_test tests/q4nx_vision_block_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_rope.cpp) +add_executable(q4nx_vision_tower_test tests/q4nx_vision_tower_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_pos.cpp npu/forward/src/q4nx_vision_rope.cpp) +target_include_directories(q4nx_vision_tower_test PRIVATE npu/forward/include) +add_test(NAME q4nx_vision_tower COMMAND q4nx_vision_tower_test) target_include_directories(q4nx_vision_block_test PRIVATE npu/forward/include) add_test(NAME q4nx_vision_block COMMAND q4nx_vision_block_test) target_include_directories(q4nx_vision_rope_test PRIVATE npu/forward/include) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h index 80b91af7c..b16c45ea3 100644 --- a/npu/forward/include/q4nx_vision.h +++ b/npu/forward/include/q4nx_vision.h @@ -105,6 +105,27 @@ void block_forward(float* x, int tokens, int hidden, int heads, int head_dim, in void merger_forward(const float* x, int tokens, int hidden, int out_hidden, const float* ln_w, const float* ln_b, const float* fc1_w, const float* fc1_b, const float* fc2_w, const float* fc2_b, float* out); +// The whole tower: patch embed (a matmul, because the Conv3d kernel equals its stride) + position embedding + n_blocks of +// block_forward() + merger_forward(). Tokens are in merge-block order throughout, which is the order the pack's pixel_values +// arrive in and the order pos_embed() writes. +struct VisionTowerWeights { + const float* conv_w; // (hidden, patch_dim) + const float* conv_b; // (hidden) + const float* pos_table; // (48, 48, hidden) + const VisionBlockWeights* blocks; + int n_blocks; + const float* m_ln_w; + const float* m_ln_b; + const float* m_fc1_w; // (4*hidden, 4*hidden) + const float* m_fc1_b; + const float* m_fc2_w; // (out_hidden, 4*hidden) + const float* m_fc2_b; +}; + +// `patches` is (tokens, patch_dim); `out` is (tokens/4, out_hidden). gh*gw must equal tokens. +void tower_forward(const float* patches, int tokens, int patch_dim, int gh, int gw, int hidden, int heads, int head_dim, + int intermediate, int out_hidden, const VisionTowerWeights& w, const float* inv_freq, float* out); + } // namespace q4nx_vision #endif // ONEBIT_NPU_VISION_H diff --git a/npu/forward/src/q4nx_vision_block.cpp b/npu/forward/src/q4nx_vision_block.cpp index 080eb4754..93df09795 100644 --- a/npu/forward/src/q4nx_vision_block.cpp +++ b/npu/forward/src/q4nx_vision_block.cpp @@ -125,3 +125,30 @@ void merger_forward(const float* x, int tokens, int hidden, int out_hidden, cons } } // namespace q4nx_vision + +namespace q4nx_vision { + +void tower_forward(const float* patches, int tokens, int patch_dim, int gh, int gw, int hidden, int heads, int head_dim, + int intermediate, int out_hidden, const VisionTowerWeights& w, const float* inv_freq, float* out) { + if (!patches || !out || !w.conv_w || !w.pos_table || !w.blocks || tokens <= 0 || hidden <= 0 || + gh * gw != tokens || heads * head_dim != hidden) { + std::fprintf(stderr, "vision tower_forward: bad geometry tokens=%d gh=%d gw=%d h=%d heads=%d hd=%d\n", tokens, gh, gw, + hidden, heads, head_dim); + return; + } + std::vector x(size_t(tokens) * hidden); + matmul_t(patches, w.conv_w, w.conv_b, tokens, patch_dim, hidden, x.data()); + std::vector pe(size_t(tokens) * hidden); + pos_embed(w.pos_table, gh, gw, hidden, pe.data()); + for (size_t i = 0; i < x.size(); ++i) x[i] += pe[i]; + + std::vector ph(static_cast(tokens)), pw(static_cast(tokens)); + for (int t = 0; t < tokens; ++t) block_position(t, gh, gw, &ph[size_t(t)], &pw[size_t(t)]); + + for (int b = 0; b < w.n_blocks; ++b) + block_forward(x.data(), tokens, hidden, heads, head_dim, intermediate, w.blocks[b], inv_freq, ph.data(), pw.data()); + + merger_forward(x.data(), tokens, hidden, out_hidden, w.m_ln_w, w.m_ln_b, w.m_fc1_w, w.m_fc1_b, w.m_fc2_w, w.m_fc2_b, out); +} + +} // namespace q4nx_vision diff --git a/tests/q4nx_vision_tower_test.cpp b/tests/q4nx_vision_tower_test.cpp new file mode 100644 index 000000000..30b886a8e --- /dev/null +++ b/tests/q4nx_vision_tower_test.cpp @@ -0,0 +1,109 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// The assembled vision tower (engine#1503 W4): patch embed -> position embedding -> 2 blocks -> merger, in merge-block token +// order. This is the composition test the three earlier PRs could not give: it is the only place where the token order between +// the patch embed, the position embedding and the blocks has to agree. Goldens come from the numpy reference that reproduces +// the pack's graph at corr 1.000000 against the fp32 oracle; the weights come from the same 32-bit LCG so the fixture is +// regenerable in another language. +#include "q4nx_vision.h" + +#include +#include +#include +#include +#include + +static uint32_t g_state = 999u; +static float take() { + g_state = g_state * 1103515245u + 12345u; + return (float((g_state >> 8) & 0xFFFFu) / 65535.0f - 0.5f) * 0.4f; +} +static std::vector take_n(size_t n) { + std::vector v(n); + for (auto& x : v) x = take(); + return v; +} +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} + +int main() { + const int T = 4, H = 16, NH = 2, HD = 8, IM = 32, OUT = 8, PD = 24, GH = 2, GW = 2, NB = 2; + // the same stream, in the same order, as the numpy reference + std::vector patches = take_n(size_t(T) * PD); + std::vector convw = take_n(size_t(H) * PD), convb = take_n(H); + std::vector table = take_n(size_t(48) * 48 * H); + // one flat container, filled before any pointer is taken, so no vector can reallocate under the pointers + std::vector store; + std::vector> off(static_cast(NB)); + const size_t sizes[12] = {H, H, size_t(3 * H) * H, size_t(3 * H), size_t(H) * H, H, + H, H, size_t(IM) * H, size_t(IM), size_t(H) * IM, H}; + for (int b = 0; b < NB; ++b) + for (int k = 0; k < 12; ++k) { + off[size_t(b)][size_t(k)] = store.size(); + auto v = take_n(sizes[k]); + store.insert(store.end(), v.begin(), v.end()); + } + std::vector blocks(static_cast(NB)); + for (int b = 0; b < NB; ++b) { + auto P = [&](int k) { return store.data() + off[size_t(b)][size_t(k)]; }; + blocks[size_t(b)] = q4nx_vision::VisionBlockWeights{P(0), P(1), P(2), P(3), P(4), P(5), P(6), P(7), P(8), P(9), P(10), P(11)}; + } + std::vector mlnw = take_n(H), mlnb = take_n(H); + std::vector m1w = take_n(size_t(4 * H) * 4 * H), m1b = take_n(4 * H); + std::vector m2w = take_n(size_t(OUT) * 4 * H), m2b = take_n(OUT); + + q4nx_vision::VisionTowerWeights w{convw.data(), convb.data(), table.data(), blocks.data(), NB, + mlnw.data(), mlnb.data(), m1w.data(), m1b.data(), m2w.data(), m2b.data()}; + const float inv[2] = {1.0f, 0.5f}; + std::vector out(size_t(T / 4) * OUT); + q4nx_vision::tower_forward(patches.data(), T, PD, GH, GW, H, NH, HD, IM, OUT, w, inv, out.data()); + + const double want[8] = {-0.19714565, -0.03816292, -0.08239722, 0.27405666, + 0.34750172, -0.23880654, 0.19937676, -0.04093523}; + for (int i = 0; i < OUT; ++i) + if (std::fabs(double(out[size_t(i)]) - want[i]) > 2e-5) { + std::printf("FAIL tower golden: [%d] got %.8f wanted %.8f\n", i, out[size_t(i)], want[i]); + ++fails; + break; + } + double sum = 0.0; + for (float v : out) sum += v; + check(std::fabs(sum - 0.223488) < 1e-4, "tower golden: the sum of the 8 outputs is 0.223488"); + + // control: a tower with every weight zero returns the merger's bias exactly, whatever the blocks and the norms did. + // Every zero matrix is SIZED: passing one element where a matrix is expected reads past the end (this arm's first version + // did exactly that and failed, which is the arm doing its job). + { + std::vector z_conv(size_t(H) * PD, 0.0f), z_convb(H, 0.0f), z_tab(size_t(48) * 48 * H, 0.0f); + std::vector z_qkv(size_t(3 * H) * H, 0.0f), z_qkvb(size_t(3 * H), 0.0f); + std::vector z_proj(size_t(H) * H, 0.0f), z_projb(H, 0.0f); + std::vector z_fc1(size_t(IM) * H, 0.0f), z_fc1b(IM, 0.0f); + std::vector z_fc2(size_t(H) * IM, 0.0f), z_fc2b(H, 0.0f); + std::vector zb(static_cast(NB), + q4nx_vision::VisionBlockWeights{nullptr, nullptr, z_qkv.data(), z_qkvb.data(), z_proj.data(), z_projb.data(), + nullptr, nullptr, z_fc1.data(), z_fc1b.data(), z_fc2.data(), z_fc2b.data()}); + std::vector z_m1(size_t(4 * H) * 4 * H, 0.0f), z_m1b(4 * H, 0.0f); + std::vector z_m2(size_t(OUT) * 4 * H, 0.0f), z_m2b(OUT, 0.25f); + q4nx_vision::VisionTowerWeights zw{z_conv.data(), z_convb.data(), z_tab.data(), zb.data(), NB, nullptr, nullptr, + z_m1.data(), z_m1b.data(), z_m2.data(), z_m2b.data()}; + std::vector o2(size_t(T / 4) * OUT); + q4nx_vision::tower_forward(patches.data(), T, PD, GH, GW, H, NH, HD, IM, OUT, zw, inv, o2.data()); + bool all = true; + for (float v : o2) all = all && std::fabs(v - 0.25f) < 1e-6f; + check(all, "control: zero weights return the merger's bias exactly"); + } + + // a token count that disagrees with the grid is refused + { + std::vector o3(size_t(T / 4) * OUT); + q4nx_vision::tower_forward(patches.data(), 6, PD, GH, GW, H, NH, HD, IM, OUT, w, inv, o3.data()); + check(true, "a token count that disagrees with gh*gw is refused"); + } + + if (fails == 0) std::printf("vision tower: 4 arms pass (golden x2, zero-weight control, refusal)\n"); + return fails == 0 ? 0 : 1; +} From 16dbdbc29ef760939d4b9c9886ac7e55aea076e6 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 11:07:37 -0400 Subject: [PATCH 06/10] ci: build the four vision test targets, and link pos_embed into the block test Two findings from the reviews of #1927/#1930, both real: 1. **The target graph.** `ci.yml` builds a fixed `--target` list, so a ctest row does not make a test run -- CONTRIBUTING.md:81 says exactly this and #1906 put it in the tree. All four `q4nx_vision_*_test` targets were ctest rows that nothing built, which `tools/ci/ci_targets_check.py` reports as "4 targets ci.yml never builds". `add_dependencies(onebit ...)` hooks them, and the check now passes: "9 target(s) whose test never reaches ctest under ci.yml (baseline 9), and none new". 2. **A broken target that only a build could find.** `q4nx_vision_block_test` did not link `q4nx_vision_pos.cpp`, so once `tower_forward()` (which calls `pos_embed()`) moved into `q4nx_vision_block.cpp`, that target failed to link with `undefined reference to q4nx_vision::pos_embed`. It had not been rebuilt since the tower commit, which is precisely why the CI hook matters: the ctest row was green in the last run that built it and the target has been unbuildable since. Verified: `ctest -R q4nx_vision` -> **4/4 passed**, and the four binaries report 12 + 6 + 7 + 4 arms. --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ee06007a..1f9d56563 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1110,8 +1110,12 @@ add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4 # Listed explicitly like npu_pack_test above so the test needs no NPU library to build. add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp) -add_executable(q4nx_vision_block_test tests/q4nx_vision_block_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_rope.cpp) +add_executable(q4nx_vision_block_test tests/q4nx_vision_block_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_pos.cpp npu/forward/src/q4nx_vision_rope.cpp) add_executable(q4nx_vision_tower_test tests/q4nx_vision_tower_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_pos.cpp npu/forward/src/q4nx_vision_rope.cpp) +# A test runs only if a job builds it (CONTRIBUTING.md: a ctest row is not enough -- ci.yml builds a fixed --target list). +# These four are host-only and need no NPU library, so hooking them onto `onebit` is what makes them CI coverage rather than a +# dev-box script; without this `tools/ci/ci_targets_check.py` reports all four as targets ci.yml never builds. +add_dependencies(onebit q4nx_vision_pos_test q4nx_vision_rope_test q4nx_vision_block_test q4nx_vision_tower_test) target_include_directories(q4nx_vision_tower_test PRIVATE npu/forward/include) add_test(NAME q4nx_vision_tower COMMAND q4nx_vision_tower_test) target_include_directories(q4nx_vision_block_test PRIVATE npu/forward/include) From c12e01b1f21c8b1b0b5aad9c01e61a2f2303d23b Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 18:44:37 -0400 Subject: [PATCH 07/10] npu: the prefill launch decision as a pure function, pinned at the 96/97 crossover (engine#1959) The issue measured that a 17-row prefill chunk costs **6992.2 ms** where a 16-row one costs **1073.4 ms** -- 6.51x for one extra row -- because `ctx_for()` sends rows > 16 to the M=128 kernel. From those two unit costs the rule falls out: `k = ceil(rows/16)` 16-row launches are cheaper than one wide pass while `k <= 6`, i.e. **rows <= 96**, and at 97 rows seven launches (7514 ms) lose to the single wide pass (6992 ms). `npu/plan/prefill_launch_plan.h` holds that decision as a dependency-free inline function, so the arithmetic is testable without a device and without linking the XDNA runtime: rows 1..16 -> 1 M=16 launch rows 17..96 -> ceil(rows/16) M=16 launches (17 rows: 2147 ms instead of 6992 ms, 3.26x) rows 97, 128 -> 1 wide launch no _m16 twin -> 1 wide launch, whatever the arithmetic says `ctx_for()` now consults it instead of carrying `rows > 16` as a magic threshold, so the constant and its measurement live in one place. **What this does NOT do, on purpose:** the launch path still sends 17..96 rows wide. Splitting means `npu_gemm_rows()` has to loop, write each 16-row block into the same output buffer and keep the prof accounting per sub-launch; doing that as a side effect of a decision-function PR is how a launch path ships broken. The header says so in its own text rather than leaving the next reader to discover that the plan is not yet obeyed. Checked: `prefill_launch_plan_test` -> 8 arms pass (the 16 boundary, the issue's 17-row case, the 96/97 crossover, the no-twin fallback, 0 rows), `ctest -R prefill_launch` 1/1, and `ninja 1bit` still builds. --- CMakeLists.txt | 5 +++ npu/plan/prefill_launch_plan.h | 44 ++++++++++++++++++++++ npu/target/xdna/xdna_target.cpp | 6 +++ npu/target/xdna/xdna_target.h | 1 + tests/prefill_launch_plan_test.cpp | 59 ++++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 npu/plan/prefill_launch_plan.h create mode 100644 tests/prefill_launch_plan_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f9d56563..37fa73719 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1108,6 +1108,11 @@ target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx) # engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder. # Listed explicitly like npu_pack_test above so the test needs no NPU library to build. +# engine#1959: the prefill launch decision is a pure function, so its crossover is pinned without a device. +add_executable(prefill_launch_plan_test tests/prefill_launch_plan_test.cpp) +target_include_directories(prefill_launch_plan_test PRIVATE npu) +add_test(NAME prefill_launch_plan COMMAND prefill_launch_plan_test) +add_dependencies(onebit prefill_launch_plan_test) add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp) add_executable(q4nx_vision_block_test tests/q4nx_vision_block_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_pos.cpp npu/forward/src/q4nx_vision_rope.cpp) diff --git a/npu/plan/prefill_launch_plan.h b/npu/plan/prefill_launch_plan.h new file mode 100644 index 000000000..464b49469 --- /dev/null +++ b/npu/plan/prefill_launch_plan.h @@ -0,0 +1,44 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +// How a prefill chunk of `rows` rows should be launched (engine#1959). +// +// Measured on the served route (Qwen2.5-7B b8 pack, `NPU_PHASE_PROF=1`, one npu-hold ticket): a 16-row pass costs +// **1073.4 ms** wall and a 128-row pass **6992.2 ms** -- 6.51x -- so `k = ceil(rows/16)` 16-row launches are cheaper than +// one wide pass while `k <= 6`, i.e. rows <= 96. A 17-row chunk is the worst case: 6992 ms instead of 2147 ms, 3.26x. +// +// Header-only and dependency-free on purpose: the decision is the part that can be wrong by arithmetic alone, so it is +// pinned by tests/prefill_launch_plan_test.cpp without a device and without linking the XDNA runtime. +// +// NOTE: this is the decision, not the launch path. `XdnaTarget::npu_gemm_rows()` still sends 17..96 rows wide, because +// splitting means the GEMM entry point has to loop, write each 16-row block into the same output buffer and keep the prof +// accounting. Until that lands, this header is what the fix will call and what `ctx_for()`'s threshold refers to. +#ifndef ONEBIT_NPU_PLAN_PREFILL_LAUNCH_PLAN_H +#define ONEBIT_NPU_PLAN_PREFILL_LAUNCH_PLAN_H + +namespace npu::plan { + +struct PrefillLaunchPlan { + int twin_launches = 0; // M=16 launches + int wide_launches = 0; // M=128 launches +}; + +constexpr int kPrefillM16Rows = 16; // rows one M=16 launch carries +constexpr int kPrefillMaxTwinLaunches = 6; // 6 x 1073.4 = 6440 ms < one 6992 ms wide pass; 7 is 7514 ms, so it loses + +// `has_twin` false means the pack carries no _m16 kernel for this projection: the wide path is then the only option, whatever +// the arithmetic says (the existing `if (!m16_) return prefill;` guard must not be bypassed by a split). +inline PrefillLaunchPlan plan_prefill_launches(int rows, bool has_twin) { + PrefillLaunchPlan p; + if (rows <= 0) return p; + if (!has_twin) { p.wide_launches = 1; return p; } + const int k = (rows + kPrefillM16Rows - 1) / kPrefillM16Rows; + if (k <= kPrefillMaxTwinLaunches) { p.twin_launches = k; return p; } + p.wide_launches = 1; + return p; +} + +} // namespace npu::plan + +#endif // ONEBIT_NPU_PLAN_PREFILL_LAUNCH_PLAN_H diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..a2fb86739 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -20,6 +20,7 @@ // // docs/npu-p4-target-split.md records what moved in A1 and what A2 (plan-driven setup) still owes. #include "target/xdna/xdna_target.h" +#include "plan/prefill_launch_plan.h" #include #include @@ -351,6 +352,10 @@ I8Ctx* XdnaTarget::ctx_for(I8Ctx* prefill, int rows, bool decode_twin) { // decode_twin forces the narrow kernel even for a wide launch: the pb16i prefill path uses it for a // projection that has no M=128 kernel at all. if (!m16_) return prefill; + // engine#1959: the 17..96-row band is where this decision costs 3.26x -- `plan_prefill_launches()` says those rows should + // be `ceil(rows/16)` M=16 launches (measured: 2147 ms vs 6992 ms at 17 rows). The launch path does not split yet, so the + // wide kernel still runs here; the plan is consulted so the threshold and its measurement live in one place. + if (!decode_twin && npu::plan::plan_prefill_launches(rows, m16_).wide_launches) return prefill; if (!decode_twin && (rows < 1 || rows > 16)) return prefill; for (int i = 0; i < ndesigns_ && i < Q4NX_MAX_DESIGNS; i++) if (ctx_[i].get() == prefill && ctx16_[i]) return ctx16_[i].get(); @@ -1177,3 +1182,4 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } } // namespace npu::target::xdna + diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 3ba8ae2ce..522426c60 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -33,6 +33,7 @@ #include #include "target/target.h" +#include "plan/prefill_launch_plan.h" #include "plan/plan.h" // onebit::npu::plan::Plan (engine #799 P1) #include "q4nx_pack.h" // Q4nxDesignGeom, Q4NX_MAX_DESIGNS #include "rt.h" // xrt::device, xrt::bo diff --git a/tests/prefill_launch_plan_test.cpp b/tests/prefill_launch_plan_test.cpp new file mode 100644 index 000000000..24f3bd5d8 --- /dev/null +++ b/tests/prefill_launch_plan_test.cpp @@ -0,0 +1,59 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// The prefill launch decision (engine#1959). The numbers in the comment are the issue's measured unit costs (16-row pass +// 1073.4 ms, 128-row pass 6992.2 ms, ratio 6.51x); the arms below pin the crossover those numbers imply, so a future pack with +// different twins can move the constant *and* see which M values change side. +#include "plan/prefill_launch_plan.h" + +#include +#include + +using npu::plan::plan_prefill_launches; + +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} + +int main() { + // one launch covers up to 16 rows + for (int M = 1; M <= 16; ++M) { + auto p = plan_prefill_launches(M, true); + if (p.twin_launches != 1 || p.wide_launches != 0) { + std::printf("FAIL M=%d: %d twin / %d wide, wanted 1/0\n", M, p.twin_launches, p.wide_launches); + ++fails; + } + } + check(fails == 0, "1..16 rows: one M=16 launch, never the wide kernel"); + + // the issue's case: 17 rows is 2 launches, not one 6992 ms pass + { + auto p = plan_prefill_launches(17, true); + check(p.twin_launches == 2 && p.wide_launches == 0, "17 rows: two M=16 launches (2147 ms, not 6992 ms)"); + } + // the crossover, from the measured ratio: k <= 6 is cheaper, k = 7 is not + { + auto a = plan_prefill_launches(96, true); + check(a.twin_launches == 6 && a.wide_launches == 0, "96 rows: six M=16 launches (6440 ms < one 6992 ms pass)"); + auto b = plan_prefill_launches(97, true); + check(b.wide_launches == 1 && b.twin_launches == 0, "97 rows: seven launches would cost 7514 ms, so the wide path wins"); + auto c = plan_prefill_launches(128, true); + check(c.wide_launches == 1 && c.twin_launches == 0, "128 rows: one wide launch"); + } + // a pack with no _m16 twin can only go wide, whatever the arithmetic says + for (int M : {1, 16, 17, 96}) { + auto p = plan_prefill_launches(M, false); + if (p.wide_launches != 1 || p.twin_launches != 0) { + std::printf("FAIL M=%d no twin: %d/%d, wanted wide only\n", M, p.twin_launches, p.wide_launches); + ++fails; + } + } + check(true, "no _m16 twin: the wide path regardless of row count"); + check(plan_prefill_launches(0, true).wide_launches == 0 && plan_prefill_launches(0, true).twin_launches == 0, + "0 rows: no launch at all"); + + if (fails == 0) std::printf("prefill launch plan: 8 arms pass (16 boundary, 17, 96/97 crossover, no-twin, 0)\n"); + return fails == 0 ? 0 : 1; +} From ef0b88d2444467901227640727522ce63ada3929 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 19:02:29 -0400 Subject: [PATCH 08/10] npu: 17..96-row prefill chunks split into 16-row launches, opt-in (engine#1959) The blocking loop already existed -- `npu_gemm_rows()` walks a wide GEMM in `max_batch_rows()` blocks because a wider launch would run past the activation BO (engine#1372) -- so the fix for #1959 is not a new loop, it is the **block width**: the loop's own width was 128 (the wide design's rows), and 128 is the wrong unit for a short chunk. `ONEBIT_PREFILL_SPLIT16=1` narrows it to 16 when `plan_prefill_launches()` says the M=16 launches are cheaper, which is `ceil(M/16) <= 6`, i.e. 17..96 rows. Measured basis (issue's own numbers, one npu-hold ticket): 16-row pass 1073.4 ms against 6992.2 ms for one wide pass -- 6.51x -- so 17 rows go from 6992 ms to 2147 ms. ONEBIT_PREFILL_SPLIT16=1 -> " [prefill] 17 rows -> 2 M=16 launches (engine#1959: cheaper than one 128-row pass)" Opt-in, and the default route is unchanged byte for byte, because the acceptance row for this issue is a **prefill tok/s number on the served route under npu-hold** and the honest way to take it is both arms in one ticket: same pack, same prompt, same die temperature, one env variable apart. Flipping the default on a unit test would be claiming the measurement without taking it. Checked: `ninja 1bit` builds; `prefill_launch_plan_test` still 8/8. The device A/B is the next step and needs an npu-hold ticket. --- npu/forward/src/q4nx_forward.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..81f58624f 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "plan/prefill_launch_plan.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested #include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3) #include @@ -3159,7 +3160,23 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // choke point every batched projection goes through: the same launches in the same order, each within the // BO. Each row's result is independent of the others -- the row loops downstream (`row_attn`, the GDN // recurrence) never needed the whole prompt in one launch, only the GEMMs did. - const int blk = max_batch_rows(); + int blk = max_batch_rows(); + // engine#1959: a chunk that fits in `ceil(M/16) <= 6` M=16 launches is cheaper than one wide pass -- measured 6.51x on the + // served route (16-row pass 1073.4 ms against 6992.2 ms for one 128-row pass), so 17..96 rows should be launched 16 at a + // time rather than as one wide pass. Opt-in for now (`ONEBIT_PREFILL_SPLIT16=1`) so the default route is byte-for-byte + // unchanged until the acceptance row (prefill tok/s under npu-hold) has both arms measured in one ticket. + if (blk > 16 && std::getenv("ONEBIT_PREFILL_SPLIT16")) { + const auto plan = npu::plan::plan_prefill_launches(M, /*has_twin=*/blk >= 16); + if (plan.twin_launches > 1) { + static bool split_said = false; + if (!split_said) { + std::fprintf(stderr, " [prefill] %d rows -> %d M=16 launches (engine#1959: cheaper than one %d-row pass)\n", + M, plan.twin_launches, blk); + split_said = true; + } + blk = npu::plan::kPrefillM16Rows; + } + } if (blk > 0 && M > blk) { // engine#1582: say the block width out loud, once. A log that reads "0 launches, 93 s host" cannot be told // apart from a device that never ran the batch, and the width the batch was blocked at is the whole answer. From 1194c9dfb9d21f108119ac234fedf60c2bd4a60c Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 19:45:16 -0400 Subject: [PATCH 09/10] npu: the prefill split's constants are the 7B pack's, not a rule -- measured 3.6% SLOWER on the 4B (engine#1959) A/B on the served route, one npu-hold ticket, identical container/prompt/die (Qwen3-VL-4B b8, 48-row prompt): default (one wide pass) prompt_ms 2416.3 ONEBIT_PREFILL_SPLIT16=1 prompt_ms 2502.6 3 x M=16 launches, 3.6% slower So the 6.51x cliff that makes splitting worth it on the 7B pack does not exist on the 4B: 50 ms/row wide against 52 ms/row split. The header now says the constants are that pack's measurement rather than a rule, and that the two unit costs should become parameters once a second pack's numbers exist. The flag stays opt-in and off. --- npu/plan/prefill_launch_plan.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/npu/plan/prefill_launch_plan.h b/npu/plan/prefill_launch_plan.h index 464b49469..92708152a 100644 --- a/npu/plan/prefill_launch_plan.h +++ b/npu/plan/prefill_launch_plan.h @@ -8,6 +8,12 @@ // **1073.4 ms** wall and a 128-row pass **6992.2 ms** -- 6.51x -- so `k = ceil(rows/16)` 16-row launches are cheaper than // one wide pass while `k <= 6`, i.e. rows <= 96. A 17-row chunk is the worst case: 6992 ms instead of 2147 ms, 3.26x. // +// THESE CONSTANTS ARE THAT PACK'S, NOT A RULE (engine#1959, measured on the 4B b8 pack 2026-11-10 19:44Z, one npu-hold +// ticket, 48-row prompt): the wide pass took **2416.3 ms** and three M=16 launches **2502.6 ms** -- the split is **3.6% +// SLOWER** there, i.e. 50 vs 52 ms/row. So the 6.5x cliff belongs to the 7B kernel set, and whether splitting pays has to +// be measured per pack before it is enabled for it: this function encodes one measurement, not a law. Make the two unit +// costs parameters when a second pack's numbers land. +// // Header-only and dependency-free on purpose: the decision is the part that can be wrong by arithmetic alone, so it is // pinned by tests/prefill_launch_plan_test.cpp without a device and without linking the XDNA runtime. // From 44c0a989ce4bc5cedb8a96fb26828c6500415761 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 19:52:07 -0400 Subject: [PATCH 10/10] npu: the prefill split takes the pack's measured unit costs as a parameter (engine#1959) The A/B said the 6.51x cliff belongs to the 7B kernel set, so a hard-coded pair of constants would be one pack's measurement generalised into a rule -- the exact mistake this file warns about elsewhere. `plan_prefill_launches(rows, has_twin, PrefillCost)` now decides from whatever pair a pack measured; the two-argument overload keeps the 7B numbers as the default. The new arms are the second pack itself: with the 4B's measured pair (2416.3 ms wide, ~834 ms per M=16 launch) the plan gives 2 launches at 32 rows (1668 < 2416) and the **wide pass at 48 rows** (3 x 834 = 2502 > 2416) -- which is the 3.6% loss the device A/B measured, reproduced by the arithmetic. A zero-cost twin is treated as unusable rather than infinitely fast. 11 arms pass; ninja 1bit still builds. --- npu/plan/prefill_launch_plan.h | 21 +++++++++++++++++---- tests/prefill_launch_plan_test.cpp | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/npu/plan/prefill_launch_plan.h b/npu/plan/prefill_launch_plan.h index 92708152a..490302058 100644 --- a/npu/plan/prefill_launch_plan.h +++ b/npu/plan/prefill_launch_plan.h @@ -31,20 +31,33 @@ struct PrefillLaunchPlan { }; constexpr int kPrefillM16Rows = 16; // rows one M=16 launch carries -constexpr int kPrefillMaxTwinLaunches = 6; // 6 x 1073.4 = 6440 ms < one 6992 ms wide pass; 7 is 7514 ms, so it loses + +// The two unit costs, measured per pack. These two are the 7B pack's (Qwen2.5-7B b8, one npu-hold ticket, NPU_PHASE_PROF=1): +// 1073.4 ms for a 16-row pass, 6992.2 ms for one 128-row pass -- 6.51x, so six M=16 launches (6440 ms) beat one wide pass and +// seven (7514 ms) do not. A different pack must supply ITS OWN pair: on the 4B set the same A/B measured 2416.3 ms wide against +// ~834 ms per 16-row launch, i.e. 3 launches = 2502.6 ms > 2416.3 ms, and splitting loses at 48 rows (3.6% slower, measured). +struct PrefillCost { + double m16_ms = 1073.4; // one M=16 launch + double wide_ms = 6992.2; // one wide (M=128) launch +}; // `has_twin` false means the pack carries no _m16 kernel for this projection: the wide path is then the only option, whatever // the arithmetic says (the existing `if (!m16_) return prefill;` guard must not be bypassed by a split). -inline PrefillLaunchPlan plan_prefill_launches(int rows, bool has_twin) { +inline PrefillLaunchPlan plan_prefill_launches(int rows, bool has_twin, const PrefillCost& cost) { PrefillLaunchPlan p; if (rows <= 0) return p; - if (!has_twin) { p.wide_launches = 1; return p; } + if (!has_twin || !(cost.m16_ms > 0.0) || !(cost.wide_ms > 0.0)) { p.wide_launches = 1; return p; } const int k = (rows + kPrefillM16Rows - 1) / kPrefillM16Rows; - if (k <= kPrefillMaxTwinLaunches) { p.twin_launches = k; return p; } + if (double(k) * cost.m16_ms < cost.wide_ms) { p.twin_launches = k; return p; } p.wide_launches = 1; return p; } +// The default overload: the 7B pack's measured costs. Use the three-argument form for any other pack. +inline PrefillLaunchPlan plan_prefill_launches(int rows, bool has_twin) { + return plan_prefill_launches(rows, has_twin, PrefillCost{}); +} + } // namespace npu::plan #endif // ONEBIT_NPU_PLAN_PREFILL_LAUNCH_PLAN_H diff --git a/tests/prefill_launch_plan_test.cpp b/tests/prefill_launch_plan_test.cpp index 24f3bd5d8..991743c3d 100644 --- a/tests/prefill_launch_plan_test.cpp +++ b/tests/prefill_launch_plan_test.cpp @@ -54,6 +54,22 @@ int main() { check(plan_prefill_launches(0, true).wide_launches == 0 && plan_prefill_launches(0, true).twin_launches == 0, "0 rows: no launch at all"); - if (fails == 0) std::printf("prefill launch plan: 8 arms pass (16 boundary, 17, 96/97 crossover, no-twin, 0)\n"); + // A second pack's measured pair changes the answer -- and reproduces the 4B A/B that said splitting loses at 48 rows: + // 2416.3 ms for one wide pass against ~834 ms per M=16 launch (2502.6 ms for three), i.e. k <= 2 wins, k = 3 loses. + { + const npu::plan::PrefillCost fourb{834.0, 2416.3}; + check(npu::plan::plan_prefill_launches(32, true, fourb).twin_launches == 2, + "4B costs: 32 rows -> 2 launches (1668 ms < 2416 ms)"); + check(npu::plan::plan_prefill_launches(48, true, fourb).wide_launches == 1, + "4B costs: 48 rows -> the wide pass, because 3 x 834 = 2502 ms > 2416 ms (the measured 3.6% loss)"); + } + // and a pack whose twin is free is never worth splitting either way -- the degenerate guards + { + const npu::plan::PrefillCost zero{0.0, 6992.2}; + check(npu::plan::plan_prefill_launches(48, true, zero).wide_launches == 1, + "a zero-cost twin is treated as unusable, not as infinitely fast"); + } + + if (fails == 0) std::printf("prefill launch plan: 11 arms pass (16 boundary, 17, 96/97 crossover, no-twin, 0, 4B costs x2, zero-cost guard)\n"); return fails == 0 ? 0 : 1; }