Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1106,6 +1106,14 @@ add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp)
target_include_directories(npu_q4nx_test PRIVATE app)
target_link_libraries(npu_q4nx_test PRIVATE onebit_npu)
add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx)
# engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder.
# Listed explicitly like npu_pack_test above so the test needs no NPU library to build.
add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp)
add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp)
target_include_directories(q4nx_vision_rope_test PRIVATE npu/forward/include)
add_test(NAME q4nx_vision_rope COMMAND q4nx_vision_rope_test)
target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include)
add_test(NAME q4nx_vision_pos COMMAND q4nx_vision_pos_test)
add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp)
target_link_libraries(npu_pack_import_test PRIVATE onebit_npu)
add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden)
Expand Down
91 changes: 91 additions & 0 deletions npu/forward/include/q4nx_vision.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Proprietary and confidential. Distributed only as compiled binaries under the
// 1bit Binary License (LICENSE-BINARY).

// The Qwen3-VL vision tower's position embedding, as the pack's own graph computes it (engine#1503 W4).
//
// The graph interpolates a learned (48, 48, C) table to the patch grid and then reorders the tokens into 2x2
// merge-block order. Both halves were established against the graph itself, not inferred:
//
// * the interpolation is bilinear with **align_corners = TRUE**, i.e. `src = c * 47 / (n - 1)`. The neighbouring
// convention `c * 48 / n` reproduces the graph's corner indices for the first eight rows and columns and then
// drifts, which is what made this look like a mysterious index map for a while: measured against the graph's own
// gather, align_corners gives corr 1.00000000 / mean|d| 5e-8, and `c*48/n` gives 0.9959.
// * the reorder is exactly one permutation: `perm = [0 1 gw gw+1 | 2 3 gw+2 gw+3 | ...]`, i.e. group g takes the
// 2x2 block at (2*(g/gw2), 2*(g%gw2)) as (h,w), (h,w+1), (h+1,w), (h+1,w+1). Verified: applying it to the
// row-major resample reproduces what the graph adds to the patch embedding at corr 1.000000 / mean|d| 5e-8.
//
// Both facts are pinned by tests/q4nx_vision_pos_test.cpp against a fixture produced by the numpy reference that
// matches the fp32 oracle at corr 1.000000, so a regression here fails a test rather than a served image.
#ifndef ONEBIT_NPU_VISION_H
#define ONEBIT_NPU_VISION_H

#include <cstddef>

namespace q4nx_vision {

// Tokens the grid holds, i.e. gh * gw. The merge-block order only exists when both extents are even (merge size 2).
constexpr int kMergeSize = 2;
constexpr int kPosTableSide = 48;

// Index into the (side*side) table that position (h, w) of a (gh, gw) grid maps to, per axis: floor(c * (side-1)/(n-1))
// and the fraction towards the next index. `side` and `n` must be >= 2; the fraction is 0 when the next index clamps.
inline int axis_index(int c, int n, int side) {
const int hi = side - 1;
const int nm1 = n > 1 ? n - 1 : 1;
int i = (c * hi) / nm1; // align_corners = true
if (i > hi) i = hi;
if (i < 0) i = 0;
return i;
}
inline float axis_frac(int c, int n, int side) {
const int hi = side - 1;
const int nm1 = n > 1 ? n - 1 : 1;
const float src = float(c) * float(hi) / float(nm1);
int i = axis_index(c, n, side);
float f = src - float(i);
if (i >= hi) f = 0.0f; // no neighbour: the second tap is the same row/col
return f;
}

// Token t of a (gh, gw) patch grid, in merge-block order, as (row-major) h and w.
inline void block_position(int t, int gh, int gw, int* h, int* w) {
const int gw2 = gw / kMergeSize;
const int b = t / (kMergeSize * kMergeSize);
const int sub = t % (kMergeSize * kMergeSize);
*h = kMergeSize * (b / gw2) + sub / kMergeSize;
*w = kMergeSize * (b % gw2) + sub % kMergeSize;
(void)gh;
}

// out[token * C + c] = bilinear(table, align_corners) at the token's (h, w), where `token` runs in merge-block order.
// `table` is row-major (side, side, C); `out` is (gh*gw, C). gh and gw must be even and >= kMergeSize.
void pos_embed(const float* table, int gh, int gw, int C, float* out);

// The tower's axial 2D rotary embedding, exactly as the graph applies it to q and k:
// ang = concat([pos_h * inv_freq, pos_w * inv_freq]) (head_dim/2 wide)
// cos = TILE(cos(ang), 2), sin = TILE(sin(ang), 2) (Lg: cat([a,a]) -- a tile, NOT an interleave)
// out = x*cos + rotate_half(x)*sin, rotate_half(u) = cat(-u[head_dim/2:], u[:head_dim/2])
// Established against the graph's own post-rope q: corr 0.999996, mean|d| 3.3e-3 on |ref| 1.15. Both details are load
// bearing and both were wrong in my first three attempts: an interleaved `repeat` gives corr 0.44 where the tile gives 0.84
// before the rest of the tower is even considered, and rotating each 32-dim half separately gives 0.99 but never 1.
// `inv_freq` is the pack's own `visual.rotary_pos_emb.inv_freq` table (head_dim/4 entries); `pos_h`/`pos_w` are the token
// positions in merge-block order (see block_position).
void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, const int* pos_w);

} // namespace q4nx_vision

#endif // ONEBIT_NPU_VISION_H
51 changes: 51 additions & 0 deletions npu/forward/src/q4nx_vision_pos.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Proprietary and confidential. Distributed only as compiled binaries under the
// 1bit Binary License (LICENSE-BINARY).

#include "q4nx_vision.h"

#include <cstdio>

namespace q4nx_vision {

void pos_embed(const float* table, int gh, int gw, int C, float* out) {
const int side = kPosTableSide;
if (!table || !out || C <= 0 || gh < kMergeSize || gw < kMergeSize ||
(gh % kMergeSize) || (gw % kMergeSize)) {
std::fprintf(stderr, "vision pos_embed: bad geometry gh=%d gw=%d C=%d\n", gh, gw, C);
return;
}
const int tokens = gh * gw;
for (int t = 0; t < tokens; ++t) {
int h = 0, w = 0;
block_position(t, gh, gw, &h, &w);
const int y0 = axis_index(h, gh, side), x0 = axis_index(w, gw, side);
const int y1 = y0 + 1 < side ? y0 + 1 : y0;
const int x1 = x0 + 1 < side ? x0 + 1 : x0;
const float fy = axis_frac(h, gh, side), fx = axis_frac(w, gw, side);
const float* p00 = table + (size_t(y0) * side + x0) * C;
const float* p01 = table + (size_t(y0) * side + x1) * C;
const float* p10 = table + (size_t(y1) * side + x0) * C;
const float* p11 = table + (size_t(y1) * side + x1) * C;
float* o = out + size_t(t) * C;
const float w00 = (1.0f - fy) * (1.0f - fx), w01 = (1.0f - fy) * fx;
const float w10 = fy * (1.0f - fx), w11 = fy * fx;
for (int c = 0; c < C; ++c)
o[c] = w00 * p00[c] + w01 * p01[c] + w10 * p10[c] + w11 * p11[c];
}
}

} // namespace q4nx_vision
50 changes: 50 additions & 0 deletions npu/forward/src/q4nx_vision_rope.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Proprietary and confidential. Distributed only as compiled binaries under the
// 1bit Binary License (LICENSE-BINARY).

#include "q4nx_vision.h"

#include <cmath>
#include <cstdio>

namespace q4nx_vision {

void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h,
const int* pos_w) {
if (!x || !inv_freq || !pos_h || !pos_w || tokens <= 0 || n_heads <= 0 || head_dim < 4 || (head_dim % 4)) {
std::fprintf(stderr, "vision rope_axial: bad geometry tokens=%d heads=%d hd=%d\n", tokens, n_heads, head_dim);
return;
}
const int nf = head_dim / 4; // frequencies per axis; the angle is 2*nf wide and is TILED to head_dim
const int half = head_dim / 2;
for (int t = 0; t < tokens; ++t) {
for (int h = 0; h < n_heads; ++h) {
float* v = x + (size_t(t) * n_heads + h) * head_dim;
for (int d = 0; d < half; ++d) {
// dims [0, nf) turn with the row position, [nf, half) with the column position
const int axis = d < nf ? 0 : 1;
const float pos = float(axis == 0 ? pos_h[t] : pos_w[t]);
const float ang = pos * inv_freq[d < nf ? d : d - nf];
const float c = std::cos(ang), s = std::sin(ang);
const float a = v[d], b = v[half + d];
v[d] = a * c - b * s; // rotate_half over the whole head_dim
v[half + d] = b * c + a * s;
}
}
}
}

} // namespace q4nx_vision
18 changes: 16 additions & 2 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1525,6 +1525,20 @@ void implicit_zp_zero(Q4& q, int bits) {
for (size_t i = 0; i < q.zero.size(); ++i) q.zero[i] -= zp * q.scale[i];
}

// The bit width, taken from the SPELLING that matched (engine#1841, second point). The rule above used to be called with a
// literal `4` on the projection path and with the node's own `bits` on the head path; nothing tied the literal to the name,
// so a later spelling like `weight_Q8` would have kept the lookup working while leaving the zero point at 2^3 instead of 2^7
// -- a wrong weight with no error anywhere. Deriving it from the suffix makes the two agree by construction, and an
// unrecognised suffix falls back to the node's value (documented here) rather than to a silent 4.
int zp_bits_for_spelling(const std::string& spelling, int node_bits) {
const auto pos = spelling.rfind("_Q");
if (pos != std::string::npos && pos + 2 < spelling.size()) {
const char d = spelling[pos + 2];
if (d >= '2' && d <= '8') return d - '0';
}
return node_bits;
}

// ── MatMulNBits → (codes, scale, zero) per 32-column group ───────────────────
Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros,
int N, int K, int block, bool swap_nibbles, bool drop_zero) {
Expand Down Expand Up @@ -3729,7 +3743,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
// one ONNX Runtime's MatMulNBits uses. Missing it here decoded every PROJECTION with zero point 0 while the
// head was decoded with 8, which is why the pack imported completely, served on the device, and answered
// gibberish on both the device and the host-GEMM path (engine#1503: "West什么呢ưởng中国联通nio…").
if (!zt) implicit_zp_zero(q, 4);
if (!zt) implicit_zp_zero(q, zp_bits_for_spelling(qw->name, 4));
// the row slice, re-tiled exactly as the whole tensor would be
Q4 part;
part.N = rows; part.K = K;
Expand Down Expand Up @@ -3855,7 +3869,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows");
Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block);
// no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes)
if (!have_zp) implicit_zp_zero(q, mm.bits);
if (!have_zp) implicit_zp_zero(q, zp_bits_for_spelling(mm.weight, mm.bits));

std::vector<uint8_t> tiled;
long long pinexact = 0;
Expand Down
7 changes: 7 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,13 @@ Q4 decode_external_tile(const uint8_t* data, size_t bytes, int N, int K, std::st
Q4 decode_matmulnbits(const uint8_t* qweight, const float* scales, const uint8_t* zeros,
int N, int K, int block, bool swap_nibbles = false, bool drop_zero = false);

// The implicit zero point 2^(bits-1) for a MatMulNBits with no zeros tensor, and the bit width its SPELLING implies
// (`weight_Q4` -> 4, `weight_Q8` -> 8, anything else -> the node's own value). Exposed because the pair is what can drift:
// the projection path used to pass a hardcoded 4 while the head path passed the node's bits, so a future `weight_Q8`
// spelling would have kept the lookup working and left the zero point at 8 instead of 128 (engine#1841).
void implicit_zp_zero(Q4& q, int bits);
int zp_bits_for_spelling(const std::string& spelling, int node_bits);

// One 64-row tile of the .fconst attention payload, as the pack stores it. Established byte-exactly
// (amd-oss/npu-re docs/FACTS-oga-pack-weight-layout.md §7.13, §7.15): the tile is 2*(K/128)+2 subvolumes
// of 2,176 B = one pair per (column half, K block of 128) plus the pair holding the writer's bias input.
Expand Down
26 changes: 26 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,31 @@ void test_canonical_copy_reusable() {
// engine#1770: the canonical copy is scratch that used to only ever accumulate -- reuse writes a new file when the source
// changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry,
// different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm.
// engine#1841 (second point): the implicit zero point's bit width must follow the SPELLING that matched, not a literal at
// one call site and the node's attribute at the other. The arms below are the drift the issue described: `weight_Q4` must
// give 4 bits (zero point 8) and a hypothetical `weight_Q8` must give 8 (zero point 128) -- the number that would have
// stayed at 8, silently, if the literal were left in place.
void test_implicit_zp_bits_follow_the_spelling() {
check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q4", 4) == 4,
"zp bits: weight_Q4 declares 4 bits");
check(pi::zp_bits_for_spelling("model.layers.0.attn.q_proj.MatMul.weight_Q8", 4) == 8,
"zp bits: weight_Q8 overrides the path's 4 -- this is the arm a hardcoded literal fails");
check(pi::zp_bits_for_spelling("model.layers.0.mlp.down_proj.MatMulNBits.qweight", 3) == 3,
"zp bits: a spelling with no _Q<d> suffix falls back to the node's own value");
check(pi::zp_bits_for_spelling("weight_Q4", 8) == 4,
"zp bits: the suffix wins over a node value for a spelled tensor");
// and the effect on the tensor itself: the zero term moves by 2^(bits-1) * scale.
pi::Q4 q;
q.N = 1; q.K = 32; q.block = 128;
q.codes.assign(32, 0); q.scale.assign(1, 0.5f); q.zero.assign(1, 0.0f);
pi::implicit_zp_zero(q, 4);
check(std::fabs(q.zero[0] + 8.0f * 0.5f) < 1e-6f, "zp: 4 bits subtracts 8 * scale = 4.0");
q.zero.assign(1, 0.0f);
pi::implicit_zp_zero(q, 8);
check(std::fabs(q.zero[0] + 128.0f * 0.5f) < 1e-6f, "zp: 8 bits subtracts 128 * scale = 64.0");
std::printf(" implicit zero point: 6 arms (spelling -> bits x4, shift x2)\n");
}

void test_evict_stale_canonical_copies() {
const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid()));
std::error_code ec;
Expand Down Expand Up @@ -3390,6 +3415,7 @@ int main(int argc, char** argv) {
test_external_container_names();
test_external_packed_genre(golden_dir.empty() ? "." : golden_dir);
test_canonical_copy_reusable(); test_evict_stale_canonical_copies();
test_implicit_zp_bits_follow_the_spelling();
test_crafted_refusals(golden_dir.empty() ? "." : golden_dir);
test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir);
test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir);
Expand Down
Loading
Loading