Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1106,6 +1106,29 @@ add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp)
target_include_directories(npu_q4nx_test PRIVATE app)
target_link_libraries(npu_q4nx_test PRIVATE onebit_npu)
add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx)
# engine#1503 W4: the vision tower's position embedding -- bilinear with align_corners, then the merge-block reorder.
# Listed explicitly like npu_pack_test above so the test needs no NPU library to build.
# engine#1959: the prefill launch decision is a pure function, so its crossover is pinned without a device.
add_executable(prefill_launch_plan_test tests/prefill_launch_plan_test.cpp)
target_include_directories(prefill_launch_plan_test PRIVATE npu)
add_test(NAME prefill_launch_plan COMMAND prefill_launch_plan_test)
add_dependencies(onebit prefill_launch_plan_test)
add_executable(q4nx_vision_pos_test tests/q4nx_vision_pos_test.cpp npu/forward/src/q4nx_vision_pos.cpp)
add_executable(q4nx_vision_rope_test tests/q4nx_vision_rope_test.cpp npu/forward/src/q4nx_vision_rope.cpp)
add_executable(q4nx_vision_block_test tests/q4nx_vision_block_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_pos.cpp npu/forward/src/q4nx_vision_rope.cpp)
add_executable(q4nx_vision_tower_test tests/q4nx_vision_tower_test.cpp npu/forward/src/q4nx_vision_block.cpp npu/forward/src/q4nx_vision_pos.cpp npu/forward/src/q4nx_vision_rope.cpp)
# A test runs only if a job builds it (CONTRIBUTING.md: a ctest row is not enough -- ci.yml builds a fixed --target list).
# These four are host-only and need no NPU library, so hooking them onto `onebit` is what makes them CI coverage rather than a
# dev-box script; without this `tools/ci/ci_targets_check.py` reports all four as targets ci.yml never builds.
add_dependencies(onebit q4nx_vision_pos_test q4nx_vision_rope_test q4nx_vision_block_test q4nx_vision_tower_test)
target_include_directories(q4nx_vision_tower_test PRIVATE npu/forward/include)
add_test(NAME q4nx_vision_tower COMMAND q4nx_vision_tower_test)
target_include_directories(q4nx_vision_block_test PRIVATE npu/forward/include)
add_test(NAME q4nx_vision_block COMMAND q4nx_vision_block_test)
target_include_directories(q4nx_vision_rope_test PRIVATE npu/forward/include)
add_test(NAME q4nx_vision_rope COMMAND q4nx_vision_rope_test)
target_include_directories(q4nx_vision_pos_test PRIVATE npu/forward/include)
add_test(NAME q4nx_vision_pos COMMAND q4nx_vision_pos_test)
add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp)
target_link_libraries(npu_pack_import_test PRIVATE onebit_npu)
add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden)
Expand Down
131 changes: 131 additions & 0 deletions npu/forward/include/q4nx_vision.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
// Copyright 2026 bong-water-water-bong. All rights reserved.
// Proprietary and confidential. Distributed only as compiled binaries under the
// 1bit Binary License (LICENSE-BINARY).

// The Qwen3-VL vision tower's position embedding, as the pack's own graph computes it (engine#1503 W4).
//
// The graph interpolates a learned (48, 48, C) table to the patch grid and then reorders the tokens into 2x2
// merge-block order. Both halves were established against the graph itself, not inferred:
//
// * the interpolation is bilinear with **align_corners = TRUE**, i.e. `src = c * 47 / (n - 1)`. The neighbouring
// convention `c * 48 / n` reproduces the graph's corner indices for the first eight rows and columns and then
// drifts, which is what made this look like a mysterious index map for a while: measured against the graph's own
// gather, align_corners gives corr 1.00000000 / mean|d| 5e-8, and `c*48/n` gives 0.9959.
// * the reorder is exactly one permutation: `perm = [0 1 gw gw+1 | 2 3 gw+2 gw+3 | ...]`, i.e. group g takes the
// 2x2 block at (2*(g/gw2), 2*(g%gw2)) as (h,w), (h,w+1), (h+1,w), (h+1,w+1). Verified: applying it to the
// row-major resample reproduces what the graph adds to the patch embedding at corr 1.000000 / mean|d| 5e-8.
//
// Both facts are pinned by tests/q4nx_vision_pos_test.cpp against a fixture produced by the numpy reference that
// matches the fp32 oracle at corr 1.000000, so a regression here fails a test rather than a served image.
#ifndef ONEBIT_NPU_VISION_H
#define ONEBIT_NPU_VISION_H

#include <cstddef>

namespace q4nx_vision {

// Tokens the grid holds, i.e. gh * gw. The merge-block order only exists when both extents are even (merge size 2).
constexpr int kMergeSize = 2;
constexpr int kPosTableSide = 48;

// Index into the (side*side) table that position (h, w) of a (gh, gw) grid maps to, per axis: floor(c * (side-1)/(n-1))
// and the fraction towards the next index. `side` and `n` must be >= 2; the fraction is 0 when the next index clamps.
inline int axis_index(int c, int n, int side) {
const int hi = side - 1;
const int nm1 = n > 1 ? n - 1 : 1;
int i = (c * hi) / nm1; // align_corners = true
if (i > hi) i = hi;
if (i < 0) i = 0;
return i;
}
inline float axis_frac(int c, int n, int side) {
const int hi = side - 1;
const int nm1 = n > 1 ? n - 1 : 1;
const float src = float(c) * float(hi) / float(nm1);
int i = axis_index(c, n, side);
float f = src - float(i);
if (i >= hi) f = 0.0f; // no neighbour: the second tap is the same row/col
return f;
}

// Token t of a (gh, gw) patch grid, in merge-block order, as (row-major) h and w.
inline void block_position(int t, int gh, int gw, int* h, int* w) {
const int gw2 = gw / kMergeSize;
const int b = t / (kMergeSize * kMergeSize);
const int sub = t % (kMergeSize * kMergeSize);
*h = kMergeSize * (b / gw2) + sub / kMergeSize;
*w = kMergeSize * (b % gw2) + sub % kMergeSize;
(void)gh;
}

// out[token * C + c] = bilinear(table, align_corners) at the token's (h, w), where `token` runs in merge-block order.
// `table` is row-major (side, side, C); `out` is (gh*gw, C). gh and gw must be even and >= kMergeSize.
void pos_embed(const float* table, int gh, int gw, int C, float* out);

// The tower's axial 2D rotary embedding, exactly as the graph applies it to q and k:
// ang = concat([pos_h * inv_freq, pos_w * inv_freq]) (head_dim/2 wide)
// cos = TILE(cos(ang), 2), sin = TILE(sin(ang), 2) (Lg: cat([a,a]) -- a tile, NOT an interleave)
// out = x*cos + rotate_half(x)*sin, rotate_half(u) = cat(-u[head_dim/2:], u[:head_dim/2])
// Established against the graph's own post-rope q: corr 0.999996, mean|d| 3.3e-3 on |ref| 1.15. Both details are load
// bearing and both were wrong in my first three attempts: an interleaved `repeat` gives corr 0.44 where the tile gives 0.84
// before the rest of the tower is even considered, and rotating each 32-dim half separately gives 0.99 but never 1.
// `inv_freq` is the pack's own `visual.rotary_pos_emb.inv_freq` table (head_dim/4 entries); `pos_h`/`pos_w` are the token
// positions in merge-block order (see block_position).
void rope_axial(float* x, int tokens, int n_heads, int head_dim, const float* inv_freq, const int* pos_h, const int* pos_w);

// One vision transformer block, host float32, in the order the graph computes it:
// h = x + proj(attn(rope(qkv(LN1(x))))) -- qkv split into three contiguous hidden-wide chunks
// y = h + fc2(gelu_tanh(fc1(LN2(h))))
// Constants that are easy to get wrong and are therefore here rather than at a call site: LayerNorm eps 1e-6, the gelu is the
// TANH approximation (config `gelu_pytorch_tanh`), the attention scale is 1/sqrt(head_dim) (the graph's 0.125 at head_dim 64)
// and it is FULL attention -- the pack's 24 Loop nodes are per-image bookkeeping, `pad` measured [0, num_patches].
// Established against the graph block by block: block 0 norm1 corr 1.000000, qkv all 3072 columns > 0.999, attention
// corr 0.999994, and the whole tower matches the fp32 oracle at corr 1.000000 (mean|d| 4.1e-5).
struct VisionBlockWeights {
const float* ln1_w;
const float* ln1_b;
const float* qkv_w; // (3*hidden, hidden)
const float* qkv_b; // (3*hidden)
const float* proj_w; // (hidden, hidden)
const float* proj_b; // (hidden)
const float* ln2_w;
const float* ln2_b;
const float* fc1_w; // (intermediate, hidden)
const float* fc1_b;
const float* fc2_w; // (hidden, intermediate)
const float* fc2_b;
};

// `x` is (tokens, hidden) in place; `pos_h`/`pos_w` are those tokens' positions in merge-block order.
void block_forward(float* x, int tokens, int hidden, int heads, int head_dim, int intermediate,
const VisionBlockWeights& w, const float* inv_freq, const int* pos_h, const int* pos_w);

// The merger: LayerNorm per token, then view (tokens/4, 4*hidden), fc1 -> gelu -> fc2 to `out_hidden` (2560 for this pack).
// `out` is (tokens/4, out_hidden).
void merger_forward(const float* x, int tokens, int hidden, int out_hidden, const float* ln_w, const float* ln_b,
const float* fc1_w, const float* fc1_b, const float* fc2_w, const float* fc2_b, float* out);

// The whole tower: patch embed (a matmul, because the Conv3d kernel equals its stride) + position embedding + n_blocks of
// block_forward() + merger_forward(). Tokens are in merge-block order throughout, which is the order the pack's pixel_values
// arrive in and the order pos_embed() writes.
struct VisionTowerWeights {
const float* conv_w; // (hidden, patch_dim)
const float* conv_b; // (hidden)
const float* pos_table; // (48, 48, hidden)
const VisionBlockWeights* blocks;
int n_blocks;
const float* m_ln_w;
const float* m_ln_b;
const float* m_fc1_w; // (4*hidden, 4*hidden)
const float* m_fc1_b;
const float* m_fc2_w; // (out_hidden, 4*hidden)
const float* m_fc2_b;
};

// `patches` is (tokens, patch_dim); `out` is (tokens/4, out_hidden). gh*gw must equal tokens.
void tower_forward(const float* patches, int tokens, int patch_dim, int gh, int gw, int hidden, int heads, int head_dim,
int intermediate, int out_hidden, const VisionTowerWeights& w, const float* inv_freq, float* out);

} // namespace q4nx_vision

#endif // ONEBIT_NPU_VISION_H
19 changes: 18 additions & 1 deletion npu/forward/src/q4nx_forward.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
// Numerics follow tools/q4nx_forward_ref.py so the two can be compared
// element-wise.
#include "q4nx_forward.h"
#include "plan/prefill_launch_plan.h"
#include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested
#include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3)
#include <algorithm>
Expand Down Expand Up @@ -3159,7 +3160,23 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N
// choke point every batched projection goes through: the same launches in the same order, each within the
// BO. Each row's result is independent of the others -- the row loops downstream (`row_attn`, the GDN
// recurrence) never needed the whole prompt in one launch, only the GEMMs did.
const int blk = max_batch_rows();
int blk = max_batch_rows();
// engine#1959: a chunk that fits in `ceil(M/16) <= 6` M=16 launches is cheaper than one wide pass -- measured 6.51x on the
// served route (16-row pass 1073.4 ms against 6992.2 ms for one 128-row pass), so 17..96 rows should be launched 16 at a
// time rather than as one wide pass. Opt-in for now (`ONEBIT_PREFILL_SPLIT16=1`) so the default route is byte-for-byte
// unchanged until the acceptance row (prefill tok/s under npu-hold) has both arms measured in one ticket.
if (blk > 16 && std::getenv("ONEBIT_PREFILL_SPLIT16")) {
const auto plan = npu::plan::plan_prefill_launches(M, /*has_twin=*/blk >= 16);
if (plan.twin_launches > 1) {
static bool split_said = false;
if (!split_said) {
std::fprintf(stderr, " [prefill] %d rows -> %d M=16 launches (engine#1959: cheaper than one %d-row pass)\n",
M, plan.twin_launches, blk);
split_said = true;
}
blk = npu::plan::kPrefillM16Rows;
}
}
if (blk > 0 && M > blk) {
// engine#1582: say the block width out loud, once. A log that reads "0 launches, 93 s host" cannot be told
// apart from a device that never ran the batch, and the width the batch was blocked at is the whole answer.
Expand Down
154 changes: 154 additions & 0 deletions npu/forward/src/q4nx_vision_block.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
// Copyright 2026 bong-water-water-bong. All rights reserved.
// Proprietary and confidential. Distributed only as compiled binaries under the
// 1bit Binary License (LICENSE-BINARY).

#include "q4nx_vision.h"

#include <cmath>
#include <cstdio>
#include <vector>

namespace q4nx_vision {
namespace {

constexpr float kLnEps = 1e-6f; // the tower's LayerNorm epsilon, measured (corr 1.000000 at 1e-6, 0.0001 off at 1e-5)

void layernorm(const float* in, const float* w, const float* b, int rows, int cols, float* out) {
for (int r = 0; r < rows; ++r) {
const float* p = in + size_t(r) * cols;
float mu = 0.0f;
for (int c = 0; c < cols; ++c) mu += p[c];
mu /= float(cols);
float var = 0.0f;
for (int c = 0; c < cols; ++c) { const float d = p[c] - mu; var += d * d; }
var /= float(cols);
const float inv = 1.0f / std::sqrt(var + kLnEps);
float* o = out + size_t(r) * cols;
for (int c = 0; c < cols; ++c) o[c] = (p[c] - mu) * inv * (w ? w[c] : 1.0f) + (b ? b[c] : 0.0f);
}
}

// y = x @ W^T + bias, x (rows, k), W (n, k), y (rows, n)
void matmul_t(const float* x, const float* W, const float* b, int rows, int k, int n, float* y) {
for (int r = 0; r < rows; ++r) {
const float* xr = x + size_t(r) * k;
for (int j = 0; j < n; ++j) {
const float* wj = W + size_t(j) * k;
float acc = b ? b[j] : 0.0f;
for (int c = 0; c < k; ++c) acc += xr[c] * wj[c];
y[size_t(r) * n + j] = acc;
}
}
}

inline float gelu_tanh(float v) {
// 0.5*v*(1 + tanh(sqrt(2/pi)*(v + 0.044715 v^3))) -- config `gelu_pytorch_tanh`
return 0.5f * v * (1.0f + std::tanh(0.7978845608f * (v + 0.044715f * v * v * v)));
}

} // namespace

void block_forward(float* x, int tokens, int hidden, int heads, int head_dim, int intermediate,
const VisionBlockWeights& w, const float* inv_freq, const int* pos_h, const int* pos_w) {
if (!x || !w.qkv_w || tokens <= 0 || hidden <= 0 || heads <= 0 || head_dim <= 0 || heads * head_dim != hidden ||
intermediate <= 0 || (head_dim % 4)) {
std::fprintf(stderr, "vision block_forward: bad geometry tokens=%d h=%d heads=%d hd=%d im=%d\n", tokens, hidden,
heads, head_dim, intermediate);
return;
}
const size_t th = size_t(tokens) * hidden;
std::vector<float> n1(th), qkv(size_t(tokens) * 3 * hidden), attn(th), prj(th), n2(th),
f1(size_t(tokens) * intermediate), f2(th);

layernorm(x, w.ln1_w, w.ln1_b, tokens, hidden, n1.data());
matmul_t(n1.data(), w.qkv_w, w.qkv_b, tokens, hidden, 3 * hidden, qkv.data());

// three contiguous hidden-wide chunks: q | k | v (the head-major reading scores 0.41 against this one's 0.99)
for (int t = 0; t < tokens; ++t) {
float* q = qkv.data() + size_t(t) * 3 * hidden;
float* k = q + hidden;
float* v = k + hidden;
rope_axial(q, 1, heads, head_dim, inv_freq, pos_h + t, pos_w + t);
rope_axial(k, 1, heads, head_dim, inv_freq, pos_h + t, pos_w + t);
(void)v;
}
const float scale = 1.0f / std::sqrt(float(head_dim));
std::vector<float> scores(static_cast<size_t>(tokens));
for (int t = 0; t < tokens; ++t) {
const float* qt = qkv.data() + size_t(t) * 3 * hidden;
for (int h = 0; h < heads; ++h) {
const float* qh = qt + size_t(h) * head_dim;
float mx = -INFINITY;
for (int s = 0; s < tokens; ++s) {
const float* kh = qkv.data() + size_t(s) * 3 * hidden + hidden + size_t(h) * head_dim;
float acc = 0.0f;
for (int d = 0; d < head_dim; ++d) acc += qh[d] * kh[d];
acc *= scale;
scores[size_t(s)] = acc;
if (acc > mx) mx = acc;
}
float sum = 0.0f;
for (int s = 0; s < tokens; ++s) { scores[size_t(s)] = std::exp(scores[size_t(s)] - mx); sum += scores[size_t(s)]; }
float* o = attn.data() + size_t(t) * hidden + size_t(h) * head_dim;
for (int d = 0; d < head_dim; ++d) o[d] = 0.0f;
for (int s = 0; s < tokens; ++s) {
const float p = scores[size_t(s)] / sum;
const float* vh = qkv.data() + size_t(s) * 3 * hidden + 2 * hidden + size_t(h) * head_dim;
for (int d = 0; d < head_dim; ++d) o[d] += p * vh[d];
}
}
}
matmul_t(attn.data(), w.proj_w, w.proj_b, tokens, hidden, hidden, prj.data());
for (size_t i = 0; i < th; ++i) x[i] += prj[i];

layernorm(x, w.ln2_w, w.ln2_b, tokens, hidden, n2.data());
matmul_t(n2.data(), w.fc1_w, w.fc1_b, tokens, hidden, intermediate, f1.data());
for (auto& v : f1) v = gelu_tanh(v);
matmul_t(f1.data(), w.fc2_w, w.fc2_b, tokens, intermediate, hidden, f2.data());
for (size_t i = 0; i < th; ++i) x[i] += f2[i];
}

void merger_forward(const float* x, int tokens, int hidden, int out_hidden, const float* ln_w, const float* ln_b,
const float* fc1_w, const float* fc1_b, const float* fc2_w, const float* fc2_b, float* out) {
if (!x || !out || tokens <= 0 || hidden <= 0 || out_hidden <= 0 || (tokens % 4)) {
std::fprintf(stderr, "vision merger_forward: bad geometry tokens=%d h=%d out=%d\n", tokens, hidden, out_hidden);
return;
}
std::vector<float> n(size_t(tokens) * hidden);
layernorm(x, ln_w, ln_b, tokens, hidden, n.data());
const int rows = tokens / 4;
const int wide = 4 * hidden; // the 2x2 merge: four tokens concatenated
std::vector<float> f1(size_t(rows) * wide);
matmul_t(n.data(), fc1_w, fc1_b, rows, wide, wide, f1.data());
for (auto& v : f1) v = gelu_tanh(v);
matmul_t(f1.data(), fc2_w, fc2_b, rows, wide, out_hidden, out);
}

} // namespace q4nx_vision

namespace q4nx_vision {

void tower_forward(const float* patches, int tokens, int patch_dim, int gh, int gw, int hidden, int heads, int head_dim,
int intermediate, int out_hidden, const VisionTowerWeights& w, const float* inv_freq, float* out) {
if (!patches || !out || !w.conv_w || !w.pos_table || !w.blocks || tokens <= 0 || hidden <= 0 ||
gh * gw != tokens || heads * head_dim != hidden) {
std::fprintf(stderr, "vision tower_forward: bad geometry tokens=%d gh=%d gw=%d h=%d heads=%d hd=%d\n", tokens, gh, gw,
hidden, heads, head_dim);
return;
}
std::vector<float> x(size_t(tokens) * hidden);
matmul_t(patches, w.conv_w, w.conv_b, tokens, patch_dim, hidden, x.data());
std::vector<float> pe(size_t(tokens) * hidden);
pos_embed(w.pos_table, gh, gw, hidden, pe.data());
for (size_t i = 0; i < x.size(); ++i) x[i] += pe[i];

std::vector<int> ph(static_cast<size_t>(tokens)), pw(static_cast<size_t>(tokens));
for (int t = 0; t < tokens; ++t) block_position(t, gh, gw, &ph[size_t(t)], &pw[size_t(t)]);

for (int b = 0; b < w.n_blocks; ++b)
block_forward(x.data(), tokens, hidden, heads, head_dim, intermediate, w.blocks[b], inv_freq, ph.data(), pw.data());

merger_forward(x.data(), tokens, hidden, out_hidden, w.m_ln_w, w.m_ln_b, w.m_fc1_w, w.m_fc1_b, w.m_fc2_w, w.m_fc2_b, out);
}

} // namespace q4nx_vision
Loading
Loading