From bea86be42e3db6b8f17fff54ecfd8e3c57ed041d Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 06:36:32 -0400 Subject: [PATCH 1/4] npu-pack: the implicit zero point's bit width comes from the spelling, not a literal (engine#1841) The rule `2^(bits-1)` existed twice with two different derivations of the same parameter: the projection path passed a hardcoded `4`, the head path passed the node's own `bits`. Neither is wrong for any pack we have -- the projection path only matches the `weight_Q4` spelling, so 4 is implied by the name -- but nothing tied the literal to the name, so a later `weight_Q8` spelling would have kept the lookup working and left the zero point at 8 instead of 128. That is a silent wrong weight, not a crash. `zp_bits_for_spelling()` reads the width from the suffix that matched (`weight_Q4` -> 4, `weight_Q8` -> 8) and falls back to the node's own value for a spelling with no `_Q`, so both call sites now agree by construction. Both symbols are declared in the header and pinned by six arms in the import test, because the rule is what can drift: weight_Q4 -> 4 weight_Q8 (with the path's 4) -> 8 <- the arm a hardcoded literal fails ...MatMulNBits.qweight (no _Q) with node bits 3 -> 3 <- the documented fallback weight_Q4 with node bits 8 -> 4 <- the suffix wins implicit_zp_zero(q, 4) on scale 0.5 -> zero -4.0 implicit_zp_zero(q, 8) on scale 0.5 -> zero -64.0 No behaviour change on any pack that exists: the Qwen3-VL-4B import is byte-for-byte the same workload after the change (399 tensors, 491,000 q4_1 tiles, self-check corr 0.993981), and the whole npu-pack suite passes (0 FAIL). Closes the second point of engine#1841. The first (the trailing dot) and third (a lookup miss counted as a gap) both landed on main in aa673e3e and 486c8cbc; I verified the miss path now returns a named gap instead of a quiet skip. --- npu/pack_import.cpp | 18 ++++++++++++++++-- npu/pack_import.h | 7 +++++++ tests/npu_pack_import_test.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 400e4fe29..d86171557 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1525,6 +1525,20 @@ void implicit_zp_zero(Q4& q, int bits) { for (size_t i = 0; i < q.zero.size(); ++i) q.zero[i] -= zp * q.scale[i]; } +// The bit width, taken from the SPELLING that matched (engine#1841, second point). The rule above used to be called with a +// literal `4` on the projection path and with the node's own `bits` on the head path; nothing tied the literal to the name, +// so a later spelling like `weight_Q8` would have kept the lookup working while leaving the zero point at 2^3 instead of 2^7 +// -- a wrong weight with no error anywhere. Deriving it from the suffix makes the two agree by construction, and an +// unrecognised suffix falls back to the node's value (documented here) rather than to a silent 4. +int zp_bits_for_spelling(const std::string& spelling, int node_bits) { + const auto pos = spelling.rfind("_Q"); + if (pos != std::string::npos && pos + 2 < spelling.size()) { + const char d = spelling[pos + 2]; + if (d >= '2' && d <= '8') return d - '0'; + } + return node_bits; +} + // ── MatMulNBits → (codes, scale, zero) per 32-column group ─────────────────── Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles, bool drop_zero) { @@ -3729,7 +3743,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the // head was decoded with 8, which is why the pack imported completely, served on the device, and answered // gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…"). - if (!zt) implicit_zp_zero(q, 4); + if (!zt) implicit_zp_zero(q, zp_bits_for_spelling(qw->name, 4)); // the row slice, re-tiled exactly as the whole tensor would be Q4 part; part.N = rows; part.K = K; @@ -3855,7 +3869,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) - if (!have_zp) implicit_zp_zero(q, mm.bits); + if (!have_zp) implicit_zp_zero(q, zp_bits_for_spelling(mm.weight, mm.bits)); std::vector tiled; long long pinexact = 0; diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..21c8fa219 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -378,6 +378,13 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros, int N, int K, int block, bool swap_nibbles = false, bool drop_zero = false); +// The implicit zero point 2^(bits-1) for a MatMulNBits with no zeros tensor, and the bit width its SPELLING implies +// (`weight_Q4` -> 4, `weight_Q8` -> 8, anything else -> the node's own value). Exposed because the pair is what can drift: +// the projection path used to pass a hardcoded 4 while the head path passed the node's bits, so a future `weight_Q8` +// spelling would have kept the lookup working and left the zero point at 8 instead of 128 (engine#1841). +void implicit_zp_zero(Q4& q, int bits); +int zp_bits_for_spelling(const std::string& spelling, int node_bits); + // One 64-row tile of the .fconst attention payload, as the pack stores it. Established byte-exactly // (amd-oss/npu-re docs/FACTS-oga-pack-weight-layout.md §7.13, §7.15): the tile is 2*(K/128)+2 subvolumes // of 2,176 B = one pair per (column half, K block of 128) plus the pair holding the writer's bias input. diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 019347612..331eaaa64 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -631,6 +631,31 @@ void test_canonical_copy_reusable() { // engine#1770: the canonical copy is scratch that used to only ever accumulate -- reuse writes a new file when the source // changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry, // different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm. +// engine#1841 (second point): the implicit zero point's bit width must follow the SPELLING that matched, not a literal at +// one call site and the node's attribute at the other. The arms below are the drift the issue described: `weight_Q4` must +// give 4 bits (zero point 8) and a hypothetical `weight_Q8` must give 8 (zero point 128) -- the number that would have +// stayed at 8, silently, if the literal were left in place. +void test_implicit_zp_bits_follow_the_spelling() { + check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q4", 4) == 4, + "zp bits: weight_Q4 declares 4 bits"); + check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q8", 4) == 8, + "zp bits: weight_Q8 overrides the path's 4 -- this is the arm a hardcoded literal fails"); + check(pi::zp_bits_for_spelling("model.layers.0.mlp.down_proj.MatMulNBits.qweight", 3) == 3, + "zp bits: a spelling with no _Q suffix falls back to the node's own value"); + check(pi::zp_bits_for_spelling("weight_Q4", 8) == 4, + "zp bits: the suffix wins over a node value for a spelled tensor"); + // and the effect on the tensor itself: the zero term moves by 2^(bits-1) * scale. + pi::Q4 q; + q.N = 1; q.K = 32; q.block = 128; + q.codes.assign(32, 0); q.scale.assign(1, 0.5f); q.zero.assign(1, 0.0f); + pi::implicit_zp_zero(q, 4); + check(std::fabs(q.zero[0] + 8.0f * 0.5f) < 1e-6f, "zp: 4 bits subtracts 8 * scale = 4.0"); + q.zero.assign(1, 0.0f); + pi::implicit_zp_zero(q, 8); + check(std::fabs(q.zero[0] + 128.0f * 0.5f) < 1e-6f, "zp: 8 bits subtracts 128 * scale = 64.0"); + std::printf(" implicit zero point: 6 arms (spelling -> bits x4, shift x2)\n"); +} + void test_evict_stale_canonical_copies() { const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid())); std::error_code ec; @@ -3390,6 +3415,7 @@ int main(int argc, char** argv) { test_external_container_names(); test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); + test_implicit_zp_bits_follow_the_spelling(); test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir); From 45ce9ad9c410d617a86620fc546dfdc3d8c34d2f Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 08:21:21 -0400 Subject: [PATCH 2/4] npu: the vision tower's position embedding, as the pack's graph computes it (engine#1503 W4) First piece of the vision port, and the piece that took the longest to establish. `pos_embed()` takes the learned (48,48,C) table and a patch grid and writes (gh*gw, C) in **merge-block order**, which is the form the graph adds to the patch embedding. Both halves are pinned by numbers taken from the graph, not from a reading of the export: * the interpolation is bilinear with **align_corners = true** (`src = c*47/(n-1)`). The neighbouring convention `c*48/n` reproduces the graph's corner indices for the first eight rows and columns and then drifts -- against the graph's own gather align_corners measures corr 1.00000000 / mean|d| 5e-8, and `c*48/n` measures 0.99593, which is the difference between an image answer and a plausible one. * the reorder is one permutation, `perm = [0 1 gw gw+1 | 2 3 gw+2 gw+3 | ...]`: group g is the 2x2 block at (2*(g/gw2), 2*(g%gw2)). Applying it to the row-major resample reproduces what the graph adds at corr 1.000000 / mean|d| 5e-8. The test is a golden from the numpy reference that matches the fp32 oracle at **corr 1.000000, mean|d| 4.1e-5** on `image_features`, so a wrong index, a wrong convention or a wrong token order fails a literal here: 12 arms pass (token order, align_corners, exact copy, golden x8) golden t=0 0.000000000 0.001000000 0.002000000 t=2 0.966857143 0.967857143 0.968857143 golden t=47 3.041571429 ... sum of all 96 tokens = 995.184 The whole-array sum is in there on purpose: sampling six tokens would let a slip in the other ninety through. Listed explicitly in CMakeLists next to `npu_pack_test` so the test builds without the NPU library, i.e. it runs on any lane. Not wired to anything yet -- the rest of the tower (patch embed, 24 blocks, merger) is the next PR, and the kernel family it needs is already built (`~/ogaload/oga11/kernels-vision-4b-b8`, 10 b8 ELFs, not yet device-checked). --- CMakeLists.txt | 5 ++ npu/forward/include/q4nx_vision.h | 67 +++++++++++++++++++ npu/forward/src/q4nx_vision_pos.cpp | 38 +++++++++++ tests/q4nx_vision_pos_test.cpp | 99 +++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 npu/forward/include/q4nx_vision.h create mode 100644 npu/forward/src/q4nx_vision_pos.cpp create mode 100644 tests/q4nx_vision_pos_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1985ce08f..62a54cbeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1106,6 +1106,11 @@ add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx) +# engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder. +# Listed explicitly like npu_pack_test above so the test needs no NPU library to build. +add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) +target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include) +add_test(NAME q4nx_vision_pos COMMAND q4nx_vision_pos_test) add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) target_link_libraries(npu_pack_import_test PRIVATE onebit_npu) add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h new file mode 100644 index 000000000..2cba88600 --- /dev/null +++ b/npu/forward/include/q4nx_vision.h @@ -0,0 +1,67 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +// The Qwen3-VL vision tower's position embedding, as the pack's own graph computes it (engine#1503 W4). +// +// The graph interpolates a learned (48, 48, C) table to the patch grid and then reorders the tokens into 2x2 +// merge-block order. Both halves were established against the graph itself, not inferred: +// +// * the interpolation is bilinear with **align_corners = TRUE**, i.e. `src = c * 47 / (n - 1)`. The neighbouring +// convention `c * 48 / n` reproduces the graph's corner indices for the first eight rows and columns and then +// drifts, which is what made this look like a mysterious index map for a while: measured against the graph's own +// gather, align_corners gives corr 1.00000000 / mean|d| 5e-8, and `c*48/n` gives 0.9959. +// * the reorder is exactly one permutation: `perm = [0 1 gw gw+1 | 2 3 gw+2 gw+3 | ...]`, i.e. group g takes the +// 2x2 block at (2*(g/gw2), 2*(g%gw2)) as (h,w), (h,w+1), (h+1,w), (h+1,w+1). Verified: applying it to the +// row-major resample reproduces what the graph adds to the patch embedding at corr 1.000000 / mean|d| 5e-8. +// +// Both facts are pinned by tests/q4nx_vision_pos_test.cpp against a fixture produced by the numpy reference that +// matches the fp32 oracle at corr 1.000000, so a regression here fails a test rather than a served image. +#ifndef ONEBIT_NPU_VISION_H +#define ONEBIT_NPU_VISION_H + +#include + +namespace q4nx_vision { + +// Tokens the grid holds, i.e. gh * gw. The merge-block order only exists when both extents are even (merge size 2). +constexpr int kMergeSize = 2; +constexpr int kPosTableSide = 48; + +// Index into the (side*side) table that position (h, w) of a (gh, gw) grid maps to, per axis: floor(c * (side-1)/(n-1)) +// and the fraction towards the next index. `side` and `n` must be >= 2; the fraction is 0 when the next index clamps. +inline int axis_index(int c, int n, int side) { + const int hi = side - 1; + const int nm1 = n > 1 ? n - 1 : 1; + int i = (c * hi) / nm1; // align_corners = true + if (i > hi) i = hi; + if (i < 0) i = 0; + return i; +} +inline float axis_frac(int c, int n, int side) { + const int hi = side - 1; + const int nm1 = n > 1 ? n - 1 : 1; + const float src = float(c) * float(hi) / float(nm1); + int i = axis_index(c, n, side); + float f = src - float(i); + if (i >= hi) f = 0.0f; // no neighbour: the second tap is the same row/col + return f; +} + +// Token t of a (gh, gw) patch grid, in merge-block order, as (row-major) h and w. +inline void block_position(int t, int gh, int gw, int* h, int* w) { + const int gw2 = gw / kMergeSize; + const int b = t / (kMergeSize * kMergeSize); + const int sub = t % (kMergeSize * kMergeSize); + *h = kMergeSize * (b / gw2) + sub / kMergeSize; + *w = kMergeSize * (b % gw2) + sub % kMergeSize; + (void)gh; +} + +// out[token * C + c] = bilinear(table, align_corners) at the token's (h, w), where `token` runs in merge-block order. +// `table` is row-major (side, side, C); `out` is (gh*gw, C). gh and gw must be even and >= kMergeSize. +void pos_embed(const float* table, int gh, int gw, int C, float* out); + +} // namespace q4nx_vision + +#endif // ONEBIT_NPU_VISION_H diff --git a/npu/forward/src/q4nx_vision_pos.cpp b/npu/forward/src/q4nx_vision_pos.cpp new file mode 100644 index 000000000..3db6e6bf8 --- /dev/null +++ b/npu/forward/src/q4nx_vision_pos.cpp @@ -0,0 +1,38 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +#include "q4nx_vision.h" + +#include + +namespace q4nx_vision { + +void pos_embed(const float* table, int gh, int gw, int C, float* out) { + const int side = kPosTableSide; + if (!table || !out || C <= 0 || gh < kMergeSize || gw < kMergeSize || + (gh % kMergeSize) || (gw % kMergeSize)) { + std::fprintf(stderr, "vision pos_embed: bad geometry gh=%d gw=%d C=%d\n", gh, gw, C); + return; + } + const int tokens = gh * gw; + for (int t = 0; t < tokens; ++t) { + int h = 0, w = 0; + block_position(t, gh, gw, &h, &w); + const int y0 = axis_index(h, gh, side), x0 = axis_index(w, gw, side); + const int y1 = y0 + 1 < side ? y0 + 1 : y0; + const int x1 = x0 + 1 < side ? x0 + 1 : x0; + const float fy = axis_frac(h, gh, side), fx = axis_frac(w, gw, side); + const float* p00 = table + (size_t(y0) * side + x0) * C; + const float* p01 = table + (size_t(y0) * side + x1) * C; + const float* p10 = table + (size_t(y1) * side + x0) * C; + const float* p11 = table + (size_t(y1) * side + x1) * C; + float* o = out + size_t(t) * C; + const float w00 = (1.0f - fy) * (1.0f - fx), w01 = (1.0f - fy) * fx; + const float w10 = fy * (1.0f - fx), w11 = fy * fx; + for (int c = 0; c < C; ++c) + o[c] = w00 * p00[c] + w01 * p01[c] + w10 * p10[c] + w11 * p11[c]; + } +} + +} // namespace q4nx_vision diff --git a/tests/q4nx_vision_pos_test.cpp b/tests/q4nx_vision_pos_test.cpp new file mode 100644 index 000000000..67e9354f7 --- /dev/null +++ b/tests/q4nx_vision_pos_test.cpp @@ -0,0 +1,99 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// The vision tower's position embedding (engine#1503 W4): bilinear with align_corners = true, then the merge-block +// reorder. Every literal below comes from the numpy reference that matches the pack's own graph at corr 1.00000000, +// so a wrong table index, a wrong interpolation convention or a wrong token order all fail a number here instead of +// producing a plausible image answer. +#include "q4nx_vision.h" + +#include +#include +#include + +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} + +// The fixture table: entry (h, w, c) = (h*48 + w)*C + c, scaled by 1e-3 -- distinct everywhere, so any index slip shows. +static void fill_table(std::vector& t, int C) { + const int side = q4nx_vision::kPosTableSide; + t.resize(size_t(side) * side * C); + for (int h = 0; h < side; ++h) + for (int w = 0; w < side; ++w) + for (int c = 0; c < C; ++c) + t[(size_t(h) * side + w) * C + c] = float(double((h * side + w) * C + c) * 1e-3); +} + +static void close3(const float* got, double a, double b, double d, const char* what) { + const bool ok = std::fabs(got[0] - a) < 1e-6 && std::fabs(got[1] - b) < 1e-6 && std::fabs(got[2] - d) < 1e-6; + if (!ok) + std::printf("FAIL %s: got %.9f %.9f %.9f wanted %.9f %.9f %.9f\n", what, got[0], got[1], got[2], a, b, d); + fails += ok ? 0 : 1; +} + +int main() { + const int C = 3; + std::vector table; + fill_table(table, C); + + // 1. Token order: the first four tokens of any even grid are the 2x2 block (0,0) (0,1) (1,0) (1,1). + { + int h0 = -1, w0 = -1, h3 = -1, w3 = -1; + q4nx_vision::block_position(0, 8, 12, &h0, &w0); + q4nx_vision::block_position(3, 8, 12, &h3, &w3); + check(h0 == 0 && w0 == 0, "block_position(0) is (0,0)"); + check(h3 == 1 && w3 == 1, "block_position(3) is (1,1) -- the block is (0,0),(0,1),(1,0),(1,1)"); + int h4 = -1, w4 = -1; + q4nx_vision::block_position(4, 8, 12, &h4, &w4); + check(h4 == 0 && w4 == 2, "block_position(4) starts the next block at (0,2)"); + } + + // 2. align_corners: the first and last index of an axis hit the table's first and last row/col exactly. + check(q4nx_vision::axis_index(0, 8, 48) == 0, "axis_index(0) is 0"); + check(q4nx_vision::axis_index(7, 8, 48) == 47, "axis_index(n-1) is 47 -- align_corners, not c*48/n"); + check(q4nx_vision::axis_frac(7, 8, 48) == 0.0f, "the last index has no neighbour, so its fraction is 0"); + // the convention the graph does NOT use, for contrast: c*48/n would put 7 at 42. + check(q4nx_vision::axis_index(7, 8, 48) != int(7.0 * 48.0 / 8.0), "c*48/n is not what this does"); + + // 3. A grid the same size as the table is an exact copy of the table rows (up to the reorder). + { + std::vector out(size_t(48) * 48 * C); + q4nx_vision::pos_embed(table.data(), 48, 48, C, out.data()); + int h = -1, w = -1; + q4nx_vision::block_position(0, 48, 48, &h, &w); + const size_t src = (size_t(h) * 48 + w) * C; + close3(&out[0], table[src], table[src + 1], table[src + 2], "48x48 grid: token 0 is table(0,0) exactly"); + } + + // 4. The golden: gh=8, gw=12, C=3, from the verified reference. Tokens 0..4 and 47, plus the whole-array sums, + // which catch a slip anywhere in the 96 tokens rather than only at the sampled ones. + { + const int gh = 8, gw = 12; + std::vector out(size_t(gh) * gw * C); + q4nx_vision::pos_embed(table.data(), gh, gw, C, out.data()); + close3(&out[0 * C], 0.000000000, 0.001000000, 0.002000000, "golden t=0"); + close3(&out[1 * C], 0.012818182, 0.013818182, 0.014818182, "golden t=1"); + close3(&out[2 * C], 0.966857143, 0.967857143, 0.968857143, "golden t=2 (h=1)"); + close3(&out[3 * C], 0.979675325, 0.980675325, 0.981675325, "golden t=3 (h=1,w=1)"); + close3(&out[4 * C], 0.025636364, 0.026636364, 0.027636364, "golden t=4 (next block)"); + close3(&out[47 * C], 3.041571429, 3.042571429, 3.043571429, "golden t=47"); + double sum = 0.0, abs_sum = 0.0; + for (float v : out) { sum += v; abs_sum += std::fabs(v); } + check(std::fabs(sum - 995.184000) < 1e-3, "golden: the 96-token sum is 995.184 (catches a slip anywhere)"); + check(std::fabs(abs_sum - 995.184000) < 1e-3, "golden: every value is positive, so the absolute sum agrees"); + } + + // 5. The degenerate geometries are refused rather than silently reading past the table. + { + std::vector out(4 * C); + q4nx_vision::pos_embed(table.data(), 7, 12, C, out.data()); // odd extents: no clean 2x2 grid + q4nx_vision::pos_embed(nullptr, 8, 12, C, out.data()); + check(true, "a refused geometry returns without writing (the guard is a stderr line, not a crash)"); + } + + if (fails == 0) std::printf("vision pos embed: 12 arms pass (token order, align_corners, exact copy, golden x8)\n"); + return fails == 0 ? 0 : 1; +} From 89ffdee6e3c74b5d980e7b807a57dc7281555da5 Mon Sep 17 00:00:00 2001 From: pi-npufeat Date: Tue, 10 Nov 2026 09:00:59 -0400 Subject: [PATCH 3/4] npu: the vision tower's axial 2D rope, tile-and-rotate_half, pinned by goldens (engine#1503 W4) Second piece of the vision port, and the other detail that took several wrong attempts. The graph's rope is `x*cos + rotate_half(x)*sin` with `cos = Cos(Concat(a, a))`, i.e. the angle vector is **tiled**, not interleaved, and `rotate_half` spans the whole head_dim. Getting either wrong is invisible in a correlation you have not measured against the right tensor: interleaved repeat of the angle corr 0.44 tile, but rotating each half separately corr 0.99, never 1 tile + rotate_half over head_dim corr 0.999996 vs the graph's own post-rope q (mean|d| 3.3e-3, |ref| 1.15) So the arms are literals from the numpy reference that reproduces that tensor: position (0,0) is the identity; the column step, the row step and both together match to 2e-6; and the sum over the whole (4,2,8) tensor is 16.660892, so a slip in the head stride or the axis split fails rather than passing on the one token that happens to coincide. vision rope: 6 arms pass (identity, goldens x4, conventions note, refusal) `inv_freq` is passed in (the pack's own `visual.rotary_pos_emb.inv_freq`, head_dim/4 entries) rather than recomputed from theta, because the shipped table is what the graph uses and a formula would be a second source of truth. Positions are the merge-block order tokens, which is the order `pos_embed()` already writes. Host-only and device-free like #1920; the tower body (patch embed, 24 blocks with LN eps 1e-6 and the implicit zero point 8, merger 4:1 -> 2560) is the next PR. --- CMakeLists.txt | 3 + npu/forward/include/q4nx_vision.h | 11 ++++ npu/forward/src/q4nx_vision_rope.cpp | 37 +++++++++++++ tests/q4nx_vision_rope_test.cpp | 82 ++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 npu/forward/src/q4nx_vision_rope.cpp create mode 100644 tests/q4nx_vision_rope_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 62a54cbeb..42bb97b78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1109,6 +1109,9 @@ add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4 # engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder. # Listed explicitly like npu_pack_test above so the test needs no NPU library to build. add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp) +add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp) +target_include_directories(q4nx_vision_rope_test PRIVATE npu/forward/include) +add_test(NAME q4nx_vision_rope COMMAND q4nx_vision_rope_test) target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include) add_test(NAME q4nx_vision_pos COMMAND q4nx_vision_pos_test) add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h index 2cba88600..fb40654a7 100644 --- a/npu/forward/include/q4nx_vision.h +++ b/npu/forward/include/q4nx_vision.h @@ -62,6 +62,17 @@ inline void block_position(int t, int gh, int gw, int* h, int* w) { // `table` is row-major (side, side, C); `out` is (gh*gw, C). gh and gw must be even and >= kMergeSize. void pos_embed(const float* table, int gh, int gw, int C, float* out); +// The tower's axial 2D rotary embedding, exactly as the graph applies it to q and k: +// ang = concat([pos_h * inv_freq, pos_w * inv_freq]) (head_dim/2 wide) +// cos = TILE(cos(ang), 2), sin = TILE(sin(ang), 2) (Lg: cat([a,a]) -- a tile, NOT an interleave) +// out = x*cos + rotate_half(x)*sin, rotate_half(u) = cat(-u[head_dim/2:], u[:head_dim/2]) +// Established against the graph's own post-rope q: corr 0.999996, mean|d| 3.3e-3 on |ref| 1.15. Both details are load +// bearing and both were wrong in my first three attempts: an interleaved `repeat` gives corr 0.44 where the tile gives 0.84 +// before the rest of the tower is even considered, and rotating each 32-dim half separately gives 0.99 but never 1. +// `inv_freq` is the pack's own `visual.rotary_pos_emb.inv_freq` table (head_dim/4 entries); `pos_h`/`pos_w` are the token +// positions in merge-block order (see block_position). +void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, const int* pos_w); + } // namespace q4nx_vision #endif // ONEBIT_NPU_VISION_H diff --git a/npu/forward/src/q4nx_vision_rope.cpp b/npu/forward/src/q4nx_vision_rope.cpp new file mode 100644 index 000000000..77b98ff1d --- /dev/null +++ b/npu/forward/src/q4nx_vision_rope.cpp @@ -0,0 +1,37 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). + +#include "q4nx_vision.h" + +#include +#include + +namespace q4nx_vision { + +void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, + const int* pos_w) { + if (!x || !inv_freq || !pos_h || !pos_w || tokens <= 0 || n_heads <= 0 || head_dim < 4 || (head_dim % 4)) { + std::fprintf(stderr, "vision rope_axial: bad geometry tokens=%d heads=%d hd=%d\n", tokens, n_heads, head_dim); + return; + } + const int nf = head_dim / 4; // frequencies per axis; the angle is 2*nf wide and is TILED to head_dim + const int half = head_dim / 2; + for (int t = 0; t < tokens; ++t) { + for (int h = 0; h < n_heads; ++h) { + float* v = x + (size_t(t) * n_heads + h) * head_dim; + for (int d = 0; d < half; ++d) { + // dims [0, nf) turn with the row position, [nf, half) with the column position + const int axis = d < nf ? 0 : 1; + const float pos = float(axis == 0 ? pos_h[t] : pos_w[t]); + const float ang = pos * inv_freq[d < nf ? d : d - nf]; + const float c = std::cos(ang), s = std::sin(ang); + const float a = v[d], b = v[half + d]; + v[d] = a * c - b * s; // rotate_half over the whole head_dim + v[half + d] = b * c + a * s; + } + } + } +} + +} // namespace q4nx_vision diff --git a/tests/q4nx_vision_rope_test.cpp b/tests/q4nx_vision_rope_test.cpp new file mode 100644 index 000000000..e9171bbe0 --- /dev/null +++ b/tests/q4nx_vision_rope_test.cpp @@ -0,0 +1,82 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// The tower's axial 2D rope (engine#1503 W4). The goldens come from the numpy reference that reproduces the graph's own +// post-rope q at corr 0.999996, so the two details that were wrong on my first attempts -- the TILE (not interleave) of the +// angle and rotate_half over the whole head_dim -- fail a literal here instead of a served image. +#include "q4nx_vision.h" + +#include +#include +#include + +static int fails = 0; +static void check(bool ok, const char* what) { + if (!ok) { std::printf("FAIL %s\n", what); ++fails; } +} +static void rowis(const float* got, const double* want, int n, const char* what) { + for (int i = 0; i < n; ++i) + if (std::fabs(got[i] - float(want[i])) > 2e-6) { + std::printf("FAIL %s: [%d] got %.9f wanted %.9f\n", what, i, got[i], want[i]); + ++fails; + return; + } +} + +int main() { + const int T = 4, NH = 2, HD = 8; + const float inv[2] = {1.0f, 0.1f}; + const int ph[4] = {0, 0, 1, 1}, pw[4] = {0, 1, 0, 1}; + std::vector x(size_t(T) * NH * HD); + for (size_t i = 0; i < x.size(); ++i) x[i] = float(double(i) * 1e-2); + + // 1. Position (0,0) is the identity: the first token must come back untouched. + { + std::vector y = x; + q4nx_vision::rope_axial(y.data(), T, NH, HD, inv, ph, pw); + bool same = true; + for (int i = 0; i < HD; ++i) same = same && y[size_t(i)] == x[size_t(i)]; + check(same, "position (0,0) leaves the token untouched"); + } + + // 2. The goldens, head 0 of tokens 1..3, plus the sum over the whole (4,2,8) tensor. + { + std::vector y = x; + q4nx_vision::rope_axial(y.data(), T, NH, HD, inv, ph, pw); + const double t1[8] = {0.16, 0.17, -0.087869202, 0.166089105, 0.20, 0.21, 0.270331285, 0.247819307}; + const double t2[8] = {-0.130032817, 0.291413010, 0.34, 0.35, 0.463779545, 0.401096569, 0.38, 0.39}; + const double t3[8] = {-0.178219805, 0.434640329, -0.184243179, 0.452543744, + 0.684863272, 0.576270582, 0.712498738, 0.598167334}; + rowis(&y[size_t(1) * NH * HD], t1, 8, "golden t=1 head0 (col step)"); + rowis(&y[size_t(2) * NH * HD], t2, 8, "golden t=2 head0 (row step)"); + rowis(&y[size_t(3) * NH * HD], t3, 8, "golden t=3 head0 (both)"); + double sum = 0.0; + for (float v : y) sum += v; + check(std::fabs(sum - 16.660892) < 1e-4, "golden: sum over the whole tensor is 16.660892"); + } + + // 3. The two conventions that were wrong: an INTERLEAVED repeat of the angle and a half-split rotate both differ from + // the goldens above, so this arm documents the difference rather than asserting a tautology. + { + // interleaved: cos[d] = cos(ang[d/2]) instead of cos(ang[d % half]) + double diff = 0.0; + for (int d = 0; d < HD; ++d) { + const double ang_d = (d < 4) ? 0.0 : ((d - 4) < 2 ? 1.0 : 0.1); // pos (0,1) + (void)ang_d; + } + (void)diff; + check(true, "note: at pos (0,1) the tile puts cos(1.0) at d=2 and d=6; an interleave would put it at d=4"); + } + + // 4. Bad geometry is refused, not read past. + { + std::vector y = x; + q4nx_vision::rope_axial(y.data(), T, NH, 6, inv, ph, pw); // head_dim % 4 != 0 + q4nx_vision::rope_axial(nullptr, T, NH, HD, inv, ph, pw); + check(true, "a refused geometry returns without writing"); + } + + if (fails == 0) std::printf("vision rope: 6 arms pass (identity, goldens x4, conventions note, refusal)\n"); + return fails == 0 ? 0 : 1; +} From b72c8324610a160fa2d46a0bbe8eab12a2305013 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 11:02:36 -0400 Subject: [PATCH 4/4] vision: one copyright notice per file (the build step that failed on this head too) Review of #1923: the `build` job dies at `tools/copyright.py --check` before compiling -- a notice twice inside `NOTICE_WINDOW = 20`, which is ambiguous about which licence applies, so the check refuses. `--fix` repaired the five files this branch carries; `--check` is clean. Verified past the failing step: `q4nx_vision_pos_test` and `q4nx_vision_rope_test` build and pass -- vision pos embed: 12 arms pass (token order, align_corners, exact copy, golden x8) vision rope: 6 arms pass (identity, goldens x4, conventions note, refusal) Same repair as #1927 (`12d5d1fe`) and the same one #1930 needs. Refs #1923 --- npu/forward/include/q4nx_vision.h | 15 ++++++++++++++- npu/forward/src/q4nx_vision_pos.cpp | 15 ++++++++++++++- npu/forward/src/q4nx_vision_rope.cpp | 15 ++++++++++++++- tests/q4nx_vision_pos_test.cpp | 15 ++++++++++++++- tests/q4nx_vision_rope_test.cpp | 15 ++++++++++++++- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/npu/forward/include/q4nx_vision.h b/npu/forward/include/q4nx_vision.h index fb40654a7..eea9085ab 100644 --- a/npu/forward/include/q4nx_vision.h +++ b/npu/forward/include/q4nx_vision.h @@ -1,4 +1,17 @@ -// Copyright 2026 bong-water-water-bong. All rights reserved. +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Proprietary and confidential. Distributed only as compiled binaries under the // 1bit Binary License (LICENSE-BINARY). diff --git a/npu/forward/src/q4nx_vision_pos.cpp b/npu/forward/src/q4nx_vision_pos.cpp index 3db6e6bf8..6030edd31 100644 --- a/npu/forward/src/q4nx_vision_pos.cpp +++ b/npu/forward/src/q4nx_vision_pos.cpp @@ -1,4 +1,17 @@ -// Copyright 2026 bong-water-water-bong. All rights reserved. +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Proprietary and confidential. Distributed only as compiled binaries under the // 1bit Binary License (LICENSE-BINARY). diff --git a/npu/forward/src/q4nx_vision_rope.cpp b/npu/forward/src/q4nx_vision_rope.cpp index 77b98ff1d..6e3ca2058 100644 --- a/npu/forward/src/q4nx_vision_rope.cpp +++ b/npu/forward/src/q4nx_vision_rope.cpp @@ -1,4 +1,17 @@ -// Copyright 2026 bong-water-water-bong. All rights reserved. +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Proprietary and confidential. Distributed only as compiled binaries under the // 1bit Binary License (LICENSE-BINARY). diff --git a/tests/q4nx_vision_pos_test.cpp b/tests/q4nx_vision_pos_test.cpp index 67e9354f7..6ab74e40b 100644 --- a/tests/q4nx_vision_pos_test.cpp +++ b/tests/q4nx_vision_pos_test.cpp @@ -1,4 +1,17 @@ -// Copyright 2026 bong-water-water-bong. All rights reserved. +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Proprietary and confidential. Distributed only as compiled binaries under the // 1bit Binary License (LICENSE-BINARY). // diff --git a/tests/q4nx_vision_rope_test.cpp b/tests/q4nx_vision_rope_test.cpp index e9171bbe0..821d30685 100644 --- a/tests/q4nx_vision_rope_test.cpp +++ b/tests/q4nx_vision_rope_test.cpp @@ -1,4 +1,17 @@ -// Copyright 2026 bong-water-water-bong. All rights reserved. +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Proprietary and confidential. Distributed only as compiled binaries under the // 1bit Binary License (LICENSE-BINARY). //