diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1727a99b2..a9ed75674 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1834,6 +1834,13 @@ add_test(NAME npu_pb16i_layout
COMMAND ${CMAKE_SOURCE_DIR}/tests/npu_pb16i_layout_test.sh
${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/npu/forward/include
${CMAKE_SOURCE_DIR}/tests/npu_pb16i_layout_test.cpp)
+# The w4g32 weight image (q4nx_w4g32.h): the large-M int4-weight prefill tier's contract -- the b4 tile
+# order, the per-32 signed magnitudes, the A pack and the int64 reference. No device; set
+# W4G32_VEC="
M K N" to also check it against a gate vector set the NPU reproduced bit-exact.
+add_test(NAME npu_w4g32_layout
+ COMMAND ${CMAKE_SOURCE_DIR}/tests/npu_w4g32_layout_test.sh
+ ${CMAKE_CXX_COMPILER} ${CMAKE_SOURCE_DIR}/npu/forward/include
+ ${CMAKE_SOURCE_DIR}/tests/npu_w4g32_layout_test.cpp)
# The t2s weight stream (q4nx_weight_layout "t2s"): the same integer-exact weights at 2.125 b/w,
# expanded on the core. Its host-side contract is the file size, so the test pins the tile against
# the numbers tools/q4nx_t2s.py measured on a real image (50,135,040 -> 23,674,880 B for gate/up).
@@ -2357,6 +2364,19 @@ if(ONEBIT_NPU)
add_test(NAME npu_dmabuf_import COMMAND npu_dmabuf_import_test "")
set_tests_properties(npu_dmabuf_import PROPERTIES SKIP_RETURN_CODE 77)
+ # The w4g32 prefill kernel (q4nx_w4g32.h, NPU_W4_PREFILL), bit-exact on the device against the CPU reference over a
+ # seed set and the edge set (|P| at the int16 bound, |C| at the int32 bound MAGMAX picks). W4G32_GATE=" M K N"
+ # names the ELF; unset it SKIPS (77). It submits to the NPU: run it inside an npu-hold ticket.
+ add_executable(npu_w4g32_gate_test tests/npu_w4g32_gate_test.cpp)
+ target_include_directories(npu_w4g32_gate_test PRIVATE ${CMAKE_SOURCE_DIR}/npu/forward/include)
+ target_link_libraries(npu_w4g32_gate_test PRIVATE onebit_npu_lane)
+ find_package(OpenMP)
+ if(OpenMP_CXX_FOUND)
+ target_link_libraries(npu_w4g32_gate_test PRIVATE OpenMP::OpenMP_CXX)
+ endif()
+ add_test(NAME npu_w4g32_gate COMMAND npu_w4g32_gate_test)
+ set_tests_properties(npu_w4g32_gate PROPERTIES SKIP_RETURN_CODE 77)
+
# The model-generic forward (npu/forward): the GGUF-on-NPU serve route
# (docs/npu.md, "Open"). q4nx_forward.cpp + its Q4NX dequant/pack/semantics
# + model loader, against the engine tree's I8Ctx header.
diff --git a/npu/forward/engine-src/npu_engine_i8ctx_inc.h b/npu/forward/engine-src/npu_engine_i8ctx_inc.h
index 1b5709504..bf792d35c 100644
--- a/npu/forward/engine-src/npu_engine_i8ctx_inc.h
+++ b/npu/forward/engine-src/npu_engine_i8ctx_inc.h
@@ -35,6 +35,7 @@
#include "q4nx_weight_layout.h" // bpos(): the container's device weight layout
#include "q4nx_pb16i.h" // the pb16i image: integer-exact ternary decode weights
+#include "q4nx_w4g32.h" // the w4g32 image: the large-M int4-weight prefill tier
#include "q4nx_t2s.h" // the t2s stream: the same weights at 2.125 bits/weight
#include "q4nx_pack.h" // q4nx_elf_arg_sizes(): args_fit_elf() checks the ELF's own widths
#include "quant_i8.h" // the A-row quantiser: scalar reference + an AVX2 path, byte-identical
@@ -116,6 +117,14 @@ struct I8Ctx {
// `_m16` ELFs are byte-identical to the one-row design), which is why the batched paths fall back
// to per-row launches. Measured 2026-11-02 on the OGA 0.5B pack's five ELFs.
bool gemv = false;
+ // Weight image "w4g32" (q4nx_w4g32.h; NPU_W4_PREFILL=1, engine HETEROMOSAIC-2): the large-M int4-weight prefill
+ // tier. The BO holds the container's own int4 codes plus a signed int8 magnitude per (32-K group, column), built at
+ // the weight-cache fill by factor() of the same assembled float B the int8 image is packed from; group_scales[l]
+ // carries the per-column cs, so dequant_only_rows() is unchanged. A is quantised exactly as the int8 path does
+ // (q4nx_quant_i8::row into a row-major scratch), then permuted into the design's per-32-row tile order.
+ bool w4g32 = false;
+ std::vector w4_rows_; // w4g32: the row-major int8 rows before pack_a()
+ int w4_written_rows_ = -1; // rows of w4_rows_ that may be non-zero (-1 = unknown: zero it all)
uint16_t* Ag = nullptr; // gemv: the bf16 activation BO, mapped
float* Cg = nullptr; // gemv: the f32 output BO, mapped
@@ -181,6 +190,7 @@ struct I8Ctx {
size_t w_bytes() const {
if (gemv) return gemv_w_bytes(KD, ND);
if (t2s) return q4nx_t2s::w_bytes(KD, ND);
+ if (w4g32) return q4nx_w4g32::w_bytes(KD, ND);
return pb16i ? q4nx_pb16i::w_bytes(KD, ND) : (size_t)KD * (size_t)ND;
}
@@ -208,6 +218,11 @@ struct I8Ctx {
"(K=%d N=%d)\n", KD, ND);
return false;
}
+ if (w4g32 && (pb16i || t2s || blocked_b || !q4nx_w4g32::fits(KD, ND) || ND % 1024 || MD % 128)) {
+ fprintf(stderr, " I8Ctx::init_elf: the w4g32 layout needs K%%128==0, N%%1024==0, M%%128==0 and no other "
+ "weight layout (M=%d K=%d N=%d)\n", MD, KD, ND);
+ return false;
+ }
if (pb16i && !t2s && !q4nx_pb16i::fits(KD, ND)) {
fprintf(stderr, " I8Ctx::init_elf: the pb16i layout needs K%%256==0 and N%%1024==0 "
"(K=%d N=%d)\n", KD, ND);
@@ -526,8 +541,41 @@ struct I8Ctx {
}
// Pack weights into an arbitrary (already-allocated) weight BO.
+ // w4g32: factor + pack the assembled B[K][N] (zero-padded to KD x ND) into `img`; col = the per-column cs. A weight
+ // that is not code x per-32 scale exactly cannot run on this tier, and there is no int8 image to fall back to inside
+ // this context, so it is a hard, named stop: unset NPU_W4_PREFILL for this model.
+ void packB_w4g32(uint8_t* img, const float* w, int K, int N, std::vector& col, double& ssum) {
+ const float* B = w;
+ std::vector pad;
+ if (K != KD || N != ND) {
+ pad.assign((size_t)KD * ND, 0.0f);
+ for (int k = 0; k < K; k++) memcpy(pad.data() + (size_t)k * ND, w + (size_t)k * N, (size_t)N * sizeof(float));
+ B = pad.data();
+ }
+ std::vector code((size_t)KD * ND);
+ std::vector sc((size_t)(KD / q4nx_w4g32::GROUP) * ND), cs((size_t)ND);
+ if (!q4nx_w4g32::factor(B, KD, ND, code.data(), sc.data()) ||
+ !q4nx_w4g32::pack(code.data(), sc.data(), KD, ND, img, cs.data())) {
+ fprintf(stderr, " I8Ctx: NPU_W4_PREFILL: a K=%d N=%d weight is not int4 code x per-32 scale exactly, so the "
+ "w4g32 tier cannot carry it; unset NPU_W4_PREFILL for this model\n", K, N);
+ std::exit(3);
+ }
+ col.assign(cs.begin(), cs.begin() + N);
+ ssum = 0;
+ for (int n = 0; n < N; n++) ssum += (double)col[(size_t)n];
+ }
+
void packB_into(xrt::bo& bo, const float* w, int K, int N,
float& sout, std::vector& col_out) {
+ if (w4g32) {
+ std::vector col;
+ double ssum = 0;
+ packB_w4g32((uint8_t*)bo.map(), w, K, N, col, ssum);
+ bo.sync(XCL_BO_SYNC_BO_TO_DEVICE);
+ col_out = std::move(col);
+ sout = (float)(ssum / N);
+ return;
+ }
auto* Bm = (int8_t*)bo.map();
memset(Bm, 0, (size_t)KD * ND);
std::vector col(N);
@@ -616,6 +664,15 @@ struct I8Ctx {
// per-tensor scale packed low-magnitude columns (Qwen3 v_proj rms
// ~0.007 vs q/k ~0.02-0.03) onto ~10 int8 levels -> ~5% output error
// that compounds over 28 layers and flips the final token.
+ if (w4g32) {
+ std::vector col;
+ double ssum = 0;
+ packB_w4g32((uint8_t*)layerB[l]->map(), w, K, N, col, ssum);
+ layerB[l]->sync(XCL_BO_SYNC_BO_TO_DEVICE);
+ group_scales[l] = std::move(col);
+ sout = (float)(ssum / N);
+ return;
+ }
auto* Bm = (int8_t*)layerB[l]->map();
memset(Bm, 0, (size_t)KD * ND);
std::vector col(N);
@@ -671,6 +728,23 @@ struct I8Ctx {
// which costs one whole-BO zero on the first launch and cannot depend on the allocation or on member order.
int a_written_rows_ = -1; // rows [0, a_written_rows_) may hold host data; -1 = nothing known yet
+ // w4g32: prepare_activation()'s zeroing rules, applied to the row-major scratch; then the whole of bA is rewritten by
+ // pack_a() (the tile order interleaves every row, so there is no narrow dirty range).
+ inline int8_t* w4_prepare(int am, int ak) {
+ if (w4_rows_.size() != (size_t)MD * KD) { w4_rows_.assign((size_t)MD * KD, 0); w4_written_rows_ = 0; }
+ if (ak != KD || w4_written_rows_ < 0) {
+ memset(w4_rows_.data(), 0, w4_rows_.size());
+ } else if (w4_written_rows_ > am) {
+ memset(w4_rows_.data() + (size_t)am * KD, 0, (size_t)(w4_written_rows_ - am) * KD);
+ }
+ w4_written_rows_ = ak != KD ? -1 : am; // a short K leaves the column tail to re-zero next time too
+ return w4_rows_.data();
+ }
+ inline void w4_finish() {
+ q4nx_w4g32::pack_a(w4_rows_.data(), KD, MD, KD, Am);
+ a_dirty(MD);
+ }
+
inline void prepare_activation(int am, int ak) {
if (ak != KD || a_written_rows_ < 0) {
// (a) a K shorter than the padded geometry leaves a column tail in rows [0, am) that the quantiser does
@@ -728,6 +802,12 @@ struct I8Ctx {
last_am_ = am;
if (gemv) { quantize_bf16(A, am, ak); return Am; }
float ais = 1.0f / ascale;
+ if (w4g32) {
+ int8_t* rows = w4_prepare(am, ak);
+ for (int m = 0; m < am; m++) q4nx_quant_i8::row(A + (size_t)m * ak, ak, ais, rows + (size_t)m * KD);
+ w4_finish();
+ return Am;
+ }
prepare_activation(am, ak);
for (int m = 0; m < am; m++)
q4nx_quant_i8::row(A + (size_t)m * ak, ak, ais, Am + (size_t)m * KD);
@@ -755,6 +835,13 @@ struct I8Ctx {
return Am;
}
last_am_ = am;
+ if (w4g32) {
+ int8_t* rows = w4_prepare(am, ak);
+ for (int m = 0; m < am; m++)
+ q4nx_quant_i8::row(A + (size_t)m * ak, ak, 1.0f / ascales[m], rows + (size_t)m * KD);
+ w4_finish();
+ return Am;
+ }
prepare_activation(am, ak);
for (int m = 0; m < am; m++)
q4nx_quant_i8::row(A + (size_t)m * ak, ak, 1.0f / ascales[m], Am + (size_t)m * KD);
diff --git a/npu/forward/include/q4nx_w4g32.h b/npu/forward/include/q4nx_w4g32.h
new file mode 100644
index 000000000..06b7a3981
--- /dev/null
+++ b/npu/forward/include/q4nx_w4g32.h
@@ -0,0 +1,254 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// q4nx_w4g32.h -- the "w4g32" weight image: the large-M int4-weight prefill GEMM's contract.
+//
+// What it carries. A q4_1 tile grid whose zero-points are all -8 * scale (every 2-D projection
+// of bonsai2-27b-NPU, checked bit-exact on 2026-11-15) is exactly
+//
+// w[k][n] = c[k][n] * s[k/32][n], c = code - 8 in [-8, 7], s a signed bf16 per 32 K.
+//
+// The image keeps c exactly and carries s as a signed int8 magnitude per (32-K group, column) times
+// one float per column:
+//
+// s[g][n] ~= mag[g][n] * cs[n], cs[n] = max_g |s[g][n]| / MAGMAX(K), mag = rint(s / cs)
+//
+// The device sums sum_g mag[g][n] * sum_{k in g} a[m][k] * c[k][n] exactly (int8 x int4 into acc32 per
+// 32-K group, |P| <= 32512 < 2^15, then int16 x int16 into acc32, then int32 C), and the host applies
+// C_f[m][n] = C[m][n] * ascale[m] * cs[n] -- the int8 path's own dequantisation with cs as the column
+// scale. MAGMAX(K) keeps |C| <= K * 127 * 8 * MAGMAX < 2^31: 127 up to K = 6144 (6,605 levels short of
+// overflow at 5120), 121 at K = 17408. The only rounding before the host's is rint(s / cs): the worst
+// element error is |c| * cs / 2 <= 8 * max|s| / (2 * MAGMAX), the same bound as the int8 per-column image
+// (round(w * 127 / amax), amax = 8 * max|s|) when MAGMAX = 127, and the RMS error is about half of it
+// (measured 0.45-0.48x on bonsai2-27b layers 0 and 3).
+//
+// Image geometry (npu-kernels npu/arena/GU-w4/layout_w4.py, the b4 layout, k tile 128). K % 128 == 0 and
+// N % 128 == 0 (the design additionally needs N % 1024 == 0). One tile per (nt = n/128, ki = k/128),
+// at byte (nt * K/128 + ki) * 8704:
+//
+// [0, 8192) int4 codes, [kb 8][nb 8][16 r][16 cc]: element (k, n) with kb = (k%128)/16, r = k%16,
+// nb = (n%128)/16, cc = n%16; e = r*16 + cc of its 16x16 block, byte block*128 + e/2,
+// even e in the low nibble, two's complement
+// [8192, 8704) int8 mag, [nb 8][g 4][16 cc], g = (k%128)/32
+//
+// Activations are int8 with one scale per row, host-packed per 32-row tile (the coalesced design's A):
+// A_packed[rt][ki][mb][kb][r][cc] = A[rt*32 + mb*4 + r][ki*128 + kb*16 + cc]
+// (rt = m/32, mb = 4-row block, r = row in it, kb = 16-K block, cc = k in it). C is int32 row-major M x N.
+//
+// The packer's input is the int8 path's own assembled float B (no second scale stream): factor() finds an exact
+// code x per-32 scale form of it or refuses.
+//
+// This header is the contract between the packer, the A pack, the host dequantisation and the kernel's
+// tap (mm_g32h.cc: codes at w + (kb*8+nb)*128, mags at w + 8192 + (nb*4+g)*16, A at (mb*8+kb)*64).
+// tests/npu_w4g32_layout_test.cpp restates it independently and pins it without a device.
+#pragma once
+
+#include
+#include
+#include
+#include
+#include
+
+#if defined(_OPENMP)
+#define Q4NX_W4G32_PAR _Pragma("omp parallel for schedule(static)")
+#else
+#define Q4NX_W4G32_PAR
+#endif
+
+namespace q4nx_w4g32 {
+
+constexpr int KT = 128; // K rows per tile (one kernel call)
+constexpr int NT = 128; // N columns per tile
+constexpr int GROUP = 32; // K rows per magnitude
+constexpr int CODE_BYTES = KT * NT / 2; // 8192
+constexpr int MAG_BYTES = (KT / GROUP) * NT; // 512
+constexpr int TILE = CODE_BYTES + MAG_BYTES; // 8704
+constexpr int MT = 32; // activation rows per A tile (per core row)
+
+inline bool fits(int K, int N) { return K > 0 && N > 0 && K % KT == 0 && N % NT == 0; }
+inline size_t w_bytes(int K, int N) { return (size_t)(K / KT) * (size_t)(N / NT) * TILE; }
+
+// Largest |mag| that keeps the device's int32 C exact for this K: K * 127 * 8 * MAGMAX < 2^31.
+inline int magmax(int K) {
+ if (K <= 0) return 0;
+ const long long m = (long long)2147483647 / ((long long)K * 127 * 8);
+ return m >= 127 ? 127 : (int)m;
+}
+
+struct offsets {
+ size_t code; // byte holding the element's nibble
+ bool high; // the HIGH nibble of that byte
+ size_t mag; // byte of its int8 magnitude
+};
+
+// Where weight element (k, n) and its group magnitude live. Undefined unless fits(K, N).
+inline offsets index(int K, int N, int k, int n) {
+ (void)N;
+ const size_t tile = ((size_t)(n / NT) * (size_t)(K / KT) + (size_t)(k / KT)) * TILE;
+ const int kk = k % KT, nn = n % NT;
+ const int kb = kk / 16, r = kk % 16, nb = nn / 16, cc = nn % 16, g = kk / GROUP;
+ const int e = r * 16 + cc;
+ offsets o;
+ o.code = tile + (size_t)(kb * 8 + nb) * 128 + (size_t)(e / 2);
+ o.high = (e & 1) != 0;
+ o.mag = tile + CODE_BYTES + (size_t)(nb * 4 + g) * 16 + (size_t)cc;
+ return o;
+}
+
+// Factor an ASSEMBLED design matrix B[k][n] (the float the int8 path packs) into code[k][n] in [-8, 7] and
+// one scale per (32-K group, column): B = code * scale exactly, element for element. The representation the
+// kernel needs is any exact one, not the container's own: for each group with a = max |B| the candidates are
+// s = +-a/q for q = 8 .. 1, and the first under which every element is an exact in-range multiple is taken (a
+// q4_1 z = -8s pack, a symmetric q4_0 pack and a ternary pack all have one; a/q is exact whenever the true
+// scale is a/q, because the correctly rounded quotient of q*s by q is s). An all-zero group gets scale 0.
+// Returns false -- and the tier must not run on this weight -- when some group has no such scale.
+inline bool factor_group(const float* B, size_t stride, int8_t* code, float* scale_out) {
+ float a = 0.0f;
+ for (int i = 0; i < GROUP; i++) {
+ const float v = B[(size_t)i * stride];
+ if (!std::isfinite(v)) return false;
+ a = std::fmax(a, std::fabs(v));
+ }
+ if (a == 0.0f) {
+ for (int i = 0; i < GROUP; i++) code[(size_t)i * stride] = 0;
+ *scale_out = 0.0f;
+ return true;
+ }
+ for (int q = 8; q >= 1; q--)
+ for (int sg = 0; sg < 2; sg++) {
+ const float s = sg ? -a / (float)q : a / (float)q;
+ bool ok = true;
+ for (int i = 0; i < GROUP && ok; i++) {
+ const float v = B[(size_t)i * stride];
+ const float c = std::nearbyint(v / s);
+ ok = c >= -8.0f && c <= 7.0f && c * s == v;
+ }
+ if (!ok) continue;
+ for (int i = 0; i < GROUP; i++) code[(size_t)i * stride] = (int8_t)std::nearbyint(B[(size_t)i * stride] / s);
+ *scale_out = s;
+ return true;
+ }
+ return false;
+}
+
+inline bool factor(const float* B, int K, int N, int8_t* code, float* scale) {
+ if (!B || !code || !scale || K % GROUP || N <= 0) return false;
+ int bad = 0;
+ Q4NX_W4G32_PAR
+ for (int g = 0; g < K / GROUP; g++) {
+ int b = 0;
+ for (int n = 0; n < N; n++)
+ b |= !factor_group(B + (size_t)g * GROUP * N + n, (size_t)N, code + (size_t)g * GROUP * N + n,
+ scale + (size_t)g * N + n);
+ if (b) {
+#if defined(_OPENMP)
+#pragma omp atomic write
+#endif
+ bad = 1;
+ }
+ }
+ return bad == 0;
+}
+
+// Build the image and the column scales from code[k][n] (in [-8, 7]) and scale[g][n]. `img` must hold
+// w_bytes(K, N). Returns false on a bad geometry or a code out of range.
+inline bool pack(const int8_t* code, const float* scale, int K, int N, uint8_t* img, float* cs) {
+ if (!fits(K, N) || !code || !scale || !img || !cs) return false;
+ const int mm = magmax(K), G = K / GROUP, KTn = K / KT, NTn = N / NT;
+ Q4NX_W4G32_PAR
+ for (int n = 0; n < N; n++) {
+ float amax = 0.0f;
+ for (int g = 0; g < G; g++) amax = std::fmax(amax, std::fabs(scale[(size_t)g * N + n]));
+ cs[n] = amax > 0.0f ? amax / (float)mm : 1.0f;
+ }
+ int bad = 0;
+ // One tile per iteration: tile (nt, ki) owns bytes [(nt*KTn + ki) * TILE, +TILE), so iterations never share
+ // a byte and the image does not depend on the schedule.
+ Q4NX_W4G32_PAR
+ for (int t = 0; t < NTn * KTn; t++) {
+ const int nt = t / KTn, ki = t % KTn;
+ uint8_t* tile = img + (size_t)t * TILE;
+ std::memset(tile, 0, TILE);
+ int b = 0;
+ for (int g = 0; g < KT / GROUP; g++)
+ for (int nn = 0; nn < NT; nn++) {
+ const int n = nt * NT + nn;
+ long q = std::lrint(scale[(size_t)(ki * (KT / GROUP) + g) * N + n] / cs[n]);
+ if (q > mm) q = mm;
+ if (q < -mm) q = -mm;
+ tile[CODE_BYTES + (nn / 16 * 4 + g) * 16 + nn % 16] = (uint8_t)(int8_t)q;
+ }
+ for (int kk = 0; kk < KT; kk++) {
+ const int8_t* row = code + (size_t)(ki * KT + kk) * N + (size_t)nt * NT;
+ const int kb = kk / 16, r = kk % 16;
+ for (int nn = 0; nn < NT; nn++) {
+ const int c = row[nn];
+ b |= (c < -8 || c > 7);
+ const int e = r * 16 + nn % 16;
+ uint8_t& byte = tile[(kb * 8 + nn / 16) * 128 + e / 2];
+ byte |= (e & 1) ? (uint8_t)((c & 0x0F) << 4) : (uint8_t)(c & 0x0F);
+ }
+ }
+ if (b) {
+#if defined(_OPENMP)
+#pragma omp atomic write
+#endif
+ bad = 1;
+ }
+ }
+ return bad == 0;
+}
+
+// Read element (k, n) back as (code, mag).
+inline void read(const uint8_t* img, int K, int N, int k, int n, int* c, int* mag) {
+ const offsets o = index(K, N, k, n);
+ const int nib = o.high ? (img[o.code] >> 4) : (img[o.code] & 0x0F);
+ *c = nib >= 8 ? nib - 16 : nib;
+ *mag = (int)(int8_t)img[o.mag];
+}
+
+// The design's A: int8 rows a[m][ld] (row-major, the int8 path's own quantised rows) -> the packed tile
+// order above. M % 32 == 0, K % 128 == 0; `out` holds M*K bytes.
+inline bool pack_a(const int8_t* a, int ld, int M, int K, int8_t* out) {
+ if (!a || !out || M % MT || K % KT || ld < K) return false;
+ size_t o = 0;
+ for (int rt = 0; rt < M / MT; rt++)
+ for (int ki = 0; ki < K / KT; ki++)
+ for (int mb = 0; mb < MT / 4; mb++)
+ for (int kb = 0; kb < KT / 16; kb++)
+ for (int r = 0; r < 4; r++) {
+ std::memcpy(out + o, a + (size_t)(rt * MT + mb * 4 + r) * ld + (size_t)(ki * KT + kb * 16), 16);
+ o += 16;
+ }
+ return true;
+}
+
+// The device's integer result for one (m, n), from the image: the CPU reference the gate compares against.
+inline int64_t ref_dot(const int8_t* a_row, const uint8_t* img, int K, int N, int n) {
+ int64_t acc = 0;
+ for (int g = 0; g < K / GROUP; g++) {
+ int64_t p = 0;
+ int mag = 0;
+ for (int kk = 0; kk < GROUP; kk++) {
+ int c;
+ read(img, K, N, g * GROUP + kk, n, &c, &mag);
+ p += (int64_t)a_row[g * GROUP + kk] * c;
+ }
+ acc += p * mag;
+ }
+ return acc;
+}
+
+} // namespace q4nx_w4g32
diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp
index c1ff4d17d..343f5949e 100644
--- a/npu/forward/src/q4nx_forward.cpp
+++ b/npu/forward/src/q4nx_forward.cpp
@@ -6672,6 +6672,12 @@ bool Q4nxNpuForward::run_lm_head_rows(const std::vector& h_rows, int M,
const int T = i_wide >= 0 ? designs_[i_wide].N : 0;
const int ntiles = T > 0 ? (NV + T - 1) / T : 0;
I8Ctx* hctx = i_wide >= 0 ? xd_->pick_ctx(i_wide, M) : nullptr;
+ // NPU_W4_PREFILL: the role's prefill context is a w4g32 one, whose kernel reads a w4g32 image -- and the head's
+ // tiles (lm_head_packed_) are int8 images. Launching them there multiplies the wrong bytes (every row's logits
+ // were garbage while all 368 layer GEMMs checked correct). The head keeps the int8 decode twin, in chunks of its
+ // width; with no twin the batched head is skipped and the per-row fallback below runs on the head context.
+ if (hctx && hctx->w4g32) hctx = xd_->pick_ctx(i_wide, M, /*decode_twin=*/true);
+ if (hctx && hctx->w4g32) hctx = nullptr;
// A verify batch may be wider than the twin's row capacity (mtp_acceptance measures up to 128 rows).
// The launch is then made in CHUNKS of at most MD rows rather than falling back to one launch per
// row: the weights are already resident, so a chunk costs one launch, and M=128 becomes 8 launches
@@ -6956,7 +6962,13 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector
// N, same per-launch cost.
// The head keeps the int8 family even in a pb16i container: lm_head is not a
// PTQ1_0 ternary tensor and has no image, and packB below is the int8 packer.
- I8Ctx& lctx = xd_->head_ctx() ? *xd_->head_ctx() : *xd_->pick_ctx(i_qkv, 1);
+ I8Ctx& lctx = xd_->head_ctx() ? *xd_->head_ctx() : *xd_->pick_ctx(i_qkv, 1, /*decode_twin=*/xd_->w4_rows() > 0);
+ // The head packs and launches int8 images: it must never sit on a w4g32 (NPU_W4_PREFILL) prefill context.
+ if (lctx.w4g32) {
+ err_ = "lm head: NPU_W4_PREFILL needs an int8 head context (an LMHEAD ELF or the _m16 decode twins); this pack "
+ "has neither, so unset NPU_W4_PREFILL";
+ return false;
+ }
// Belt for the gemv family: packB_into writes a KD*ND int8 image, which is larger than that
// family's tile-stream BO. The head context above handles this family; reaching here means a
// container with no head ELF, and a loud refusal beats a heap overrun (measured: it aborted a
diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp
index 622710d1a..9e6f125f6 100644
--- a/npu/target/xdna/xdna_target.cpp
+++ b/npu/target/xdna/xdna_target.cpp
@@ -130,6 +130,38 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string&
err = "open_prefill: no design set (" + std::to_string(n) + " designs)";
return false;
}
+ // NPU_W4_PREFILL=1: the large-M int4-weight prefill tier (q4nx_w4g32.h). Each int8 prefill role is opened on
+ // "_w4g32_m.elf" instead, from the same directory, when EVERY such role has one at the same M (all
+ // roles or none: max_rows() is the chunk width, and a mixed set would launch some roles past their buffers). A
+ // missing file is logged and the set stays int8 -- an optional speed-up never stops a model loading. Not for the
+ // pb16i/t2s/b8 layouts, whose images are not the int8 path's float B.
+ w4_rows_ = 0;
+ std::vector w4_elf((size_t)n);
+ if (const char* w4 = std::getenv("NPU_W4_PREFILL"); w4 && std::atoi(w4) != 0 && !host_gemm) {
+ std::string why;
+ int rows = 0;
+ if (b8 || pb16i_) why = "the container's weight layout is not row-major int8";
+ for (int i = 0; why.empty() && i < n; i++) {
+ if (role_is_pb16i_split(i)) continue;
+ std::string stem(designs[i].elf_name);
+ if (stem.size() > 4 && stem.compare(stem.size() - 4, 4, ".elf") == 0) stem.resize(stem.size() - 4);
+ int got = 0;
+ for (int m : {512, 256, 128}) {
+ const std::string f = dir + "/" + stem + "_w4g32_m" + std::to_string(m) + ".elf";
+ if (elf_present(f)) { got = m; w4_elf[(size_t)i] = f; break; }
+ }
+ if (!got) why = "no " + stem + "_w4g32_m<512|256|128>.elf in " + dir;
+ else if (rows && got != rows) why = "the roles' w4g32 kernels disagree on M (" + std::to_string(rows) + " vs " +
+ std::to_string(got) + ")";
+ rows = got;
+ }
+ if (why.empty() && rows > 0) {
+ w4_rows_ = rows;
+ std::fprintf(stderr, " [forward] w4g32 prefill tier: every prefill role at M=%d (NPU_W4_PREFILL)\n", rows);
+ } else {
+ std::fprintf(stderr, " [forward] w4g32 prefill tier NOT adopted (%s); prefill stays int8\n", why.c_str());
+ }
+ }
for (int i = 0; i < n; i++) {
const Q4nxDesignGeom* g = &designs[i];
// The two full-attention-QKV-split designs exist only as pb16i kernels (the role has no int8
@@ -137,7 +169,7 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string&
// is built from the _pb16i ELF by the caller's twin pass.
adopt_geometry(i, *g);
if (role_is_pb16i_split(i)) continue;
- const std::string elf = dir + "/" + g->elf_name;
+ const std::string elf = w4_rows_ ? w4_elf[(size_t)i] : dir + "/" + g->elf_name;
if (host_gemm) {
// Offline mode: host GEMMs are plain float matmuls over the decoded weights, so no ELF is
// opened and none is required. A model whose kernel set is not built yet can still be
@@ -185,8 +217,9 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string&
err = "cannot create a context for role " + std::to_string(i);
return false;
}
- int md = g->M;
- {
+ int md = w4_rows_ ? w4_rows_ : g->M;
+ if (!w4_rows_) {
+ // engine#2599 (the w4g32 tier opens its own M=512 ELF instead, so this does not apply to it)
std::string twin(g->elf_name);
if (twin.size() > 4 && twin.compare(twin.size() - 4, 4, ".elf") == 0) twin.resize(twin.size() - 4);
twin = dir + "/" + twin + "_m16.elf";
@@ -200,6 +233,7 @@ bool XdnaTarget::open_prefill(const Q4nxDesignGeom* designs, int n, std::string&
}
ctx->MD = md; ctx->KD = g->K; ctx->ND = g->N;
ctx->blocked_b = b8;
+ ctx->w4g32 = w4_rows_ > 0;
if (!ctx->init_elf(*dev_, elf.c_str(), md, g->K, g->N, 1)) {
// engine#2434: carry the runtime's reason, so a refused context ("code 102 ... No such device") reaches
// open_with_retry's queue test and the lock-holder note instead of reading as a generic init failure.
@@ -245,8 +279,10 @@ bool XdnaTarget::open(const onebit::npu::plan::Plan& plan, const std::string& ke
}
// engine#2599: a pack whose "prefill" ELFs are byte-identical to the decode twins states the design geometry's M in
// its plan (bonsai2-27b-NPU: prefill.M = 128) but its kernels compute 16 rows; open_prefill() said so by name, and
- // launching at the kernel's real width is the fix, not a disagreement with the plan.
- if (plan_.prefill_m > 0 && adopted_prefill > 0 && plan_.prefill_m != adopted_prefill && !prefill_is_decode_design_)
+ // launching at the kernel's real width is the fix, not a disagreement with the plan. The w4g32 tier is a runtime
+ // choice (NPU_W4_PREFILL) over the plan's int8 set, at its own M: not a disagreement either.
+ if (plan_.prefill_m > 0 && adopted_prefill > 0 && plan_.prefill_m != adopted_prefill && !prefill_is_decode_design_ &&
+ adopted_prefill != w4_rows_)
{ err = "plan: prefill M=" + std::to_string(plan_.prefill_m) + " but the adopted kernels run M=" +
std::to_string(adopted_prefill); return false; }
if (plan_.decode_enabled && plan_.decode_m > 0 && adopted_decode > 0 &&
@@ -323,6 +359,30 @@ bool XdnaTarget::weight_cached(int layer, int key) const {
wcache_[(size_t)layer].find(key) != wcache_[(size_t)layer].end();
}
+bool XdnaTarget::fill_int8_and_w4(PackedWeight& pw, I8Ctx& kctx, int role, const float* Bmat, int K, int N,
+ std::string& err) {
+ const bool in_range = role >= 0 && role < Q4NX_MAX_DESIGNS;
+ I8Ctx* c8 = !kctx.w4g32 ? &kctx : (in_range && ctx16_[role] ? ctx16_[role].get() : nullptr);
+ I8Ctx* c4 = kctx.w4g32 ? &kctx
+ : (w4_rows_ > 0 && in_range && ctx_[role] && ctx_[role]->w4g32 ? ctx_[role].get() : nullptr);
+ float sout = 1.0f;
+ if (c8) {
+ pw.bo = c8->make_weight_bo(*dev_);
+ c8->packB_into(*pw.bo, Bmat, K, N, sout, pw.scales);
+ }
+ if (c4) {
+ pw.bo4 = c4->make_weight_bo(*dev_);
+ c4->packB_into(*pw.bo4, Bmat, K, N, sout, pw.scales4);
+ }
+ if (!c8 && !c4) { err = "weight cache: no context to pack this weight for"; return false; }
+ return true;
+}
+
+xrt::bo* XdnaTarget::slot_bo(PackedWeight& pw, const I8Ctx& kctx) { return kctx.w4g32 ? pw.bo4.get() : pw.bo.get(); }
+const std::vector* XdnaTarget::slot_scales(const PackedWeight& pw, const I8Ctx& kctx) {
+ return kctx.w4g32 ? &pw.scales4 : &pw.scales;
+}
+
int XdnaTarget::max_rows() const {
/* engine#1900/#1903: MIN over the loaded per-role widths, never the first populated one. This width IS the chunk
* width, and the activation BO holds exactly that many rows -- an over-wide launch reads and poisons past the BO,
@@ -1109,8 +1169,8 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri
// `prefill` row would hide. It must be zero for a warm process.
q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD);
PackedWeight pw;
- pw.bo = kctx.make_weight_bo(*dev_);
if (kctx.gemv) {
+ pw.bo = kctx.make_weight_bo(*dev_);
// The gemv family dequantises from the container's own tiles: upload them unchanged.
if (!ar.tiles) { err = "gemv: the model has no tile stream for this container"; return; }
std::vector blob;
@@ -1118,15 +1178,15 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri
if (!kctx.upload_tiles_into(*pw.bo, blob.data(), blob.size())) {
err = "gemv: tile upload failed"; return;
}
- } else {
- float sout = 1.0f;
- kctx.packB_into(*pw.bo, Bmat, K, N, sout, pw.scales);
+ } else if (!fill_int8_and_w4(pw, kctx, ar.role, Bmat, K, N, err)) {
+ return;
}
fill_ms = std::chrono::duration(std::chrono::steady_clock::now() - f0).count();
it = m.emplace(wkey, std::move(pw)).first;
}
slot = &it->second;
- gs = &slot->scales;
+ if (!slot_bo(*slot, kctx)) { err = "weight cache: the filled slot has no image for this context's layout"; return; }
+ gs = slot_scales(*slot, kctx);
} else if (kctx.gemv) {
err = "gemv: this family needs the weight cache (NPU_NO_WCACHE=1 is not supported with it)";
return;
@@ -1139,7 +1199,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri
const auto tl = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
auto issue = [&]() -> xrt::run {
- return slot ? kctx.launch_async_with_bo(*slot->bo, A, 1, K, as)
+ return slot ? kctx.launch_async_with_bo(*slot_bo(*slot, kctx), A, 1, K, as)
: kctx.sync_and_launch(0);
};
xrt::run r = issue();
@@ -1174,7 +1234,7 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri
}
if (m16_check_ && &kctx != &ctx)
- m16_check_gemm(ctx, kctx, A, K, N, Bmat, slot ? slot->bo.get() : nullptr, as);
+ m16_check_gemm(ctx, kctx, A, K, N, Bmat, slot ? slot_bo(*slot, kctx) : nullptr, as);
if (prof) {
const double dpk = std::chrono::duration(tl - tp).count();
@@ -1399,10 +1459,8 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std:
const auto f0 = std::chrono::steady_clock::now(); // a weight-cache fill: one-time, not per launch
q4nx_lh::LhScope _tlh_pk(LH_PACK_UPLOAD);
PackedWeight pw;
- pw.bo = kctx.make_weight_bo(*dev_);
- float sout = 1.0f;
const auto w0 = std::chrono::steady_clock::now(); // engine#1416: the 22 ms candidate
- kctx.packB_into(*pw.bo, Bmat, K, N, sout, pw.scales);
+ if (!fill_int8_and_w4(pw, kctx, ar.role, Bmat, K, N, err)) return;
const auto w1 = std::chrono::steady_clock::now();
wmiss_n_++;
wmiss_ms_ += std::chrono::duration(w1 - w0).count();
@@ -1413,7 +1471,8 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std:
it = m.emplace(wkey, std::move(pw)).first;
}
slot = &it->second;
- gs = &slot->scales;
+ if (!slot_bo(*slot, kctx)) { err = "weight cache: the filled slot has no image for this context's layout"; return; }
+ gs = slot_scales(*slot, kctx);
} else {
float sout = 1.0f;
const auto w0 = std::chrono::steady_clock::now(); // engine#1416: uncached branch
@@ -1428,7 +1487,7 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std:
}
const auto tl = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
auto issue = [&]() -> xrt::run {
- return slot ? kctx.launch_async_with_bo_rows(*slot->bo, A, M, K, ascales.data())
+ return slot ? kctx.launch_async_with_bo_rows(*slot_bo(*slot, kctx), A, M, K, ascales.data())
: kctx.launch_async_rows(0, A, M, K, ascales.data());
};
xrt::run r = issue();
@@ -1467,8 +1526,25 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std:
o[n] = (float)cm[n] * asc * (*gs)[(size_t)n];
}
if (m16_check_ && &kctx != &ctx)
- m16_check_gemm_rows(ctx, kctx, A, M, K, N, slot ? slot->bo.get() : nullptr,
+ m16_check_gemm_rows(ctx, kctx, A, M, K, N, slot ? slot_bo(*slot, kctx) : nullptr,
ascales);
+ // NPU_W4_CHECK=1: the w4g32 launch against the float reference sum_k A[m][k] * Bmat[k][n] on a sample of rows and
+ // columns, layers 0-1 only; one line per launch with the worst relative error. A diagnostic, off by default.
+ if (Bmat && ar.layer >= 0 && std::getenv("NPU_W4_CHECK")) {
+ const int astr = (ar.a_k > 0 && ar.a_k < K) ? ar.a_k : K;
+ double worst = 0, refn = 0, errn = 0;
+ for (int m : {0, std::min(5, M - 1), std::min(15, M - 1)}) {
+ for (int n = 0; n < N; n += std::max(1, N / 61)) {
+ double ref = 0;
+ for (int k = 0; k < K; k++) ref += (double)A[(size_t)m * astr + k] * (double)Bmat[(size_t)k * N + n];
+ const double got = out[(size_t)m * N + n];
+ refn += ref * ref; errn += (got - ref) * (got - ref);
+ worst = std::max(worst, std::fabs(got - ref));
+ }
+ }
+ fprintf(stderr, "[w4check] %s layer %d role %d M=%d K=%d N=%d rel-err %.4f worst-abs %.4g (ref rms %.4g)\n", kctx.w4g32 ? "w4" : "i8", ar.layer,
+ ar.role, M, K, N, std::sqrt(errn / std::max(refn, 1e-30)), worst, std::sqrt(refn / 183));
+ }
if (prof) {
const double dpk = std::chrono::duration(tl - tp).count();
const double dql = std::chrono::duration(tq - tl).count();
diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h
index 91e821fcf..a09817471 100644
--- a/npu/target/xdna/xdna_target.h
+++ b/npu/target/xdna/xdna_target.h
@@ -50,6 +50,10 @@ struct PackedWeight {
// pb16i: the sidecar's host side table. Empty eref = an int8 pack.
std::vector eref;
int pb16i_mode = 0;
+ // NPU_W4_PREFILL: the same weight as the w4g32 image (the prefill contexts) beside the int8 one (the decode twins).
+ // Both come from the one Bmat a fill sees, so weight_cached() stays true for both families at once.
+ std::unique_ptr bo4;
+ std::vector scales4;
};
// Per-role dispatch accounting for NPU_PROFILE=1.
@@ -153,6 +157,13 @@ class XdnaTarget final : public NpuTarget {
// the message a user sees is the same one the forward produced before this moved (an empty string
// means no plan, i.e. a purely derived set).
bool open_prefill(const Q4nxDesignGeom* designs, int n, std::string& err);
+ int w4_rows() const { return w4_rows_; } // 0 unless every prefill role runs the w4g32 tier
+ // A weight-cache fill for the int8 families: the int8 image for the role's int8 context (kctx itself, or its decode
+ // twin when kctx is the w4g32 one) and, with the w4g32 tier on, the w4g32 image for its prefill context.
+ bool fill_int8_and_w4(PackedWeight& pw, I8Ctx& kctx, int role, const float* Bmat, int K, int N, std::string& err);
+ // The image and column scales `kctx` launches on, from a filled slot (nullptr when the slot has none for it).
+ static xrt::bo* slot_bo(PackedWeight& pw, const I8Ctx& kctx);
+ static const std::vector* slot_scales(const PackedWeight& pw, const I8Ctx& kctx);
// The lm-head context (engine#803 A2b, the last setup block): pb16i ternary images first, then the int8
// ladder, then the q4_1 gemv head -- the body moved verbatim out of Q4nxNpuForward::open_lm_head_context.
@@ -249,6 +260,7 @@ class XdnaTarget final : public NpuTarget {
std::unique_ptr ctx16_[Q4NX_MAX_DESIGNS];
std::unique_ptr ctxw_[Q4NX_MAX_DESIGNS]; // the wide pb16i/t2s family (engine#1990)
int wide_rows_ = 0; // 0 = no wide family
+ int w4_rows_ = 0; // NPU_W4_PREFILL: the w4g32 prefill tier's M (0 = off)
bool prefill_is_decode_design_ = false; // engine#2599: the prefill ELFs are the _m16 artifacts
std::unique_ptr ctxn_[Q4NX_MAX_DESIGNS]; // the narrow pb16i/t2s family (decode)
int narrow_rows_ = 0; // 0 = no narrow family
diff --git a/tests/npu_w4g32_gate_test.cpp b/tests/npu_w4g32_gate_test.cpp
new file mode 100644
index 000000000..06dfc593a
--- /dev/null
+++ b/tests/npu_w4g32_gate_test.cpp
@@ -0,0 +1,139 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// npu_w4g32_gate_test -- the w4g32 prefill kernel, bit-exact on the device against the CPU reference.
+//
+// W4G32_GATE=" M K N" (e.g. ".../full_i8_Z_K5120_N6144_w4g32_m512.elf 512 5120 6144")
+//
+// Unset, the test SKIPS (77): a box without the ELF or the device records a skip, not a red row. Run it inside an
+// npu-hold ticket. Two weight sets, both built through q4nx_w4g32.h exactly as the engine builds an image:
+// seed random codes in [-8, 7], signed random per-32 scales, random int8 activations;
+// edge every code -8, every scale +max, every activation +127: each per-32 partial at |P| = 32*127*8 = 32512 (the
+// int16 narrowing bound inside the kernel) and |C| at K*127*8*MAGMAX(K), the int32 bound MAGMAX is chosen for.
+// Each set runs three times with C poisoned (0x5a) before every launch, and every element of every run must equal the
+// int64 reference computed from the packed image (q4nx_w4g32::read), with A taken through pack_a() from row-major rows.
+#include "q4nx_w4g32.h"
+
+#include "rt.h" // the engine's own NPU runtime (ONEBIT_NPU_RT_DIR): the same launch API the forward's I8Ctx uses
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace w = q4nx_w4g32;
+
+// The reference, from the image the device reads: C[m][n] = sum_g mag[g][n] * sum_{k in g} a[m][k] * code[k][n].
+static void reference(const std::vector& a, const std::vector& img, int M, int K, int N,
+ std::vector& C) {
+ std::vector code((size_t)K * N);
+ std::vector mag((size_t)(K / 32) * N);
+ for (int k = 0; k < K; k++)
+ for (int n = 0; n < N; n++) {
+ int c, m;
+ w::read(img.data(), K, N, k, n, &c, &m);
+ code[(size_t)k * N + n] = (int8_t)c;
+ if (k % 32 == 0) mag[(size_t)(k / 32) * N + n] = (int16_t)m;
+ }
+ C.assign((size_t)M * N, 0);
+#if defined(_OPENMP)
+#pragma omp parallel for schedule(static)
+#endif
+ for (int m = 0; m < M; m++) {
+ std::vector acc((size_t)N, 0);
+ std::vector p((size_t)N);
+ for (int g = 0; g < K / 32; g++) {
+ std::fill(p.begin(), p.end(), 0);
+ for (int kk = 0; kk < 32; kk++) {
+ const int av = a[(size_t)m * K + g * 32 + kk];
+ const int8_t* row = code.data() + (size_t)(g * 32 + kk) * N;
+ for (int n = 0; n < N; n++) p[(size_t)n] += av * row[n];
+ }
+ const int16_t* mg = mag.data() + (size_t)g * N;
+ for (int n = 0; n < N; n++) acc[(size_t)n] += (int64_t)p[(size_t)n] * mg[n];
+ }
+ for (int n = 0; n < N; n++) C[(size_t)m * N + n] = (int32_t)acc[(size_t)n]; // |acc| < 2^31 by MAGMAX
+ }
+}
+
+int main() {
+ const char* spec = std::getenv("W4G32_GATE");
+ if (!spec || !*spec) {
+ std::printf("npu_w4g32_gate: W4G32_GATE=\" M K N\" unset -- skipped\n");
+ return 77;
+ }
+ char elf[4096];
+ int M = 0, K = 0, N = 0;
+ if (std::sscanf(spec, "%4095s %d %d %d", elf, &M, &K, &N) != 4 || M % 128 || !w::fits(K, N) || N % 1024) {
+ std::printf("npu_w4g32_gate: bad W4G32_GATE \"%s\" (M %% 128, K %% 128, N %% 1024)\n", spec);
+ return 1;
+ }
+ int fails = 0;
+ try {
+ xrt::device dev{0u};
+ xrt::elf e(elf);
+ xrt::hw_context hc(dev, e);
+ xrt::ext::kernel kern(hc, "main:seq");
+ const size_t ab = (size_t)M * K, wb = w::w_bytes(K, N), cb = (size_t)M * N * 4;
+ xrt::ext::bo ba(dev, ab), bw(dev, wb), bc(dev, cb);
+ for (int set = 0; set < 2; set++) {
+ const bool edge = set == 1;
+ std::mt19937 rng(20261115u + (unsigned)set);
+ std::vector code((size_t)K * N), arow((size_t)M * K);
+ std::vector sc((size_t)(K / 32) * N);
+ for (auto& c : code) c = edge ? (int8_t)-8 : (int8_t)((int)(rng() % 16) - 8);
+ for (auto& s : sc) s = edge ? 1.0f : (float)((int)(rng() % 255) - 127) * (1.0f / 1024.0f);
+ for (auto& x : arow) x = edge ? (int8_t)127 : (int8_t)((int)(rng() % 255) - 127);
+ std::vector img(wb);
+ std::vector cs((size_t)N);
+ if (!w::pack(code.data(), sc.data(), K, N, img.data(), cs.data())) { std::printf("pack refused\n"); return 1; }
+ std::vector ap(ab);
+ w::pack_a(arow.data(), K, M, K, ap.data());
+ std::vector ref;
+ reference(arow, img, M, K, N, ref);
+ int32_t rmax = 0;
+ for (int32_t v : ref) rmax = std::max(rmax, v < 0 ? -v : v);
+ std::memcpy(ba.map(), ap.data(), ab);
+ std::memcpy(bw.map(), img.data(), wb);
+ ba.sync(XCL_BO_SYNC_BO_TO_DEVICE);
+ bw.sync(XCL_BO_SYNC_BO_TO_DEVICE);
+ for (int rep = 0; rep < 3; rep++) {
+ std::memset(bc.map(), 0x5a, cb);
+ bc.sync(XCL_BO_SYNC_BO_TO_DEVICE);
+ xrt::run r = kern(ba, bw, bc); // (A, W, C), as I8Ctx::sync_and_launch
+ const auto st = r.wait(std::chrono::milliseconds(30000));
+ bc.sync(XCL_BO_SYNC_BO_FROM_DEVICE);
+ const int32_t* c = bc.map();
+ size_t bad = 0, first = 0;
+ for (size_t i = 0; i < (size_t)M * N; i++)
+ if (c[i] != ref[i]) { if (!bad) first = i; ++bad; }
+ std::printf(" %s rep %d state=%d mismatches=%zu/%zu max|C|=%d%s\n", edge ? "edge" : "seed", rep, (int)st,
+ bad, (size_t)M * N, rmax,
+ bad ? (" first m=" + std::to_string(first / N) + " n=" + std::to_string(first % N) +
+ " dev=" + std::to_string(c[first]) + " ref=" + std::to_string(ref[first])).c_str() : "");
+ fails += bad != 0;
+ }
+ }
+ } catch (const std::exception& ex) {
+ std::printf("npu_w4g32_gate: device error: %s\n", ex.what());
+ return 1;
+ }
+ std::printf("npu_w4g32_gate: %s (%s M=%d K=%d N=%d)\n", fails ? "FAIL" : "PASS, bit-exact x3 on seed and edge", elf, M,
+ K, N);
+ return fails ? 1 : 0;
+}
diff --git a/tests/npu_w4g32_layout_test.cpp b/tests/npu_w4g32_layout_test.cpp
new file mode 100644
index 000000000..428ef4edf
--- /dev/null
+++ b/tests/npu_w4g32_layout_test.cpp
@@ -0,0 +1,265 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// npu_w4g32_layout_test -- the w4g32 weight image contract (q4nx_w4g32.h), checked without a device.
+//
+// The packed image goes straight into the weight BO, so a header that disagrees with the kernel's tap
+// multiplies the wrong pairs and the output is silently garbage. This test restates the contract
+// from the b4 layout's own definition (npu-kernels layout_w4.py: codes [nt][ki][kb][nb][r][cc] from
+// [ki][kb][r][nt][nb][cc]; mags [nt][ki][nb][g][cc]) rather than from the header:
+//
+// 1. the image size, MAGMAX(K) at the model's K values, and the int32 bound MAGMAX guarantees;
+// 2. index() is a bijection: every code nibble is claimed once, every magnitude byte by exactly 32
+// elements of one column, and both agree with the restated layout element for element;
+// 3. factor() recovers (c, s) from an assembled B = c * s and S, and refuses each way the input can
+// fail to be that (S not constant over a group, B/S not an integer, |c| out of int4, B != 0 at S == 0);
+// 4. pack()/read() round-trip the codes, and the magnitudes are rint(s / cs) with cs = max|s| / MAGMAX;
+// 5. pack_a() is the restated A order;
+// 6. ref_dot() equals the dense int64 sum over (code, mag);
+// 7. optional, when W4G32_VEC=" M K N" names a gate vector set (W.bin, packed A.bin, C_ref.bin
+// from npu-kernels W4-g128/gen_q4.py, the set the NPU reproduced bit-exact): ref_dot() on that W
+// and A reproduces C_ref, so this header reads the device's own bytes the way the device does.
+#include "q4nx_w4g32.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace w = q4nx_w4g32;
+
+static int fails = 0;
+#define CHECK(cond, ...) \
+ do { \
+ if (!(cond)) { \
+ std::printf("FAIL %s:%d: ", __FILE__, __LINE__); \
+ std::printf(__VA_ARGS__); \
+ std::printf("\n"); \
+ ++fails; \
+ } \
+ } while (0)
+
+// layout_w4.py, restated: byte of the code of (k, n) and byte of its magnitude.
+static size_t restated_code_byte(int K, int k, int n, bool* high) {
+ const int KTt = K / 128;
+ const int ki = k / 128, kb = (k % 128) / 16, r = k % 16, nt = n / 128, nb = (n % 128) / 16, cc = n % 16;
+ // flat index in [nt][ki][kb][nb][r][cc] over the code region of a tile, then the tile base
+ const size_t e = ((((size_t)kb * 8 + nb) * 16 + r) * 16 + cc);
+ *high = (e & 1) != 0;
+ return ((size_t)nt * KTt + ki) * 8704 + e / 2;
+}
+static size_t restated_mag_byte(int K, int k, int n) {
+ const int KTt = K / 128;
+ const int ki = k / 128, g = (k % 128) / 32, nt = n / 128, nb = (n % 128) / 16, cc = n % 16;
+ return ((size_t)nt * KTt + ki) * 8704 + 8192 + ((size_t)nb * 4 + g) * 16 + cc;
+}
+
+static void test_sizes() {
+ CHECK(w::TILE == 8704 && w::CODE_BYTES == 8192, "tile %d", w::TILE);
+ CHECK(w::w_bytes(5120, 17408) == 47349760u, "U image %zu", w::w_bytes(5120, 17408)); // = gen_q4's W
+ CHECK(w::magmax(5120) == 127 && w::magmax(6144) == 127, "magmax 5120/6144");
+ CHECK(w::magmax(17408) == 121, "magmax(17408) = %d", w::magmax(17408));
+ for (int K = 128; K <= 65536; K += 128) {
+ const long long worst = (long long)K * 127 * 8 * w::magmax(K);
+ CHECK(worst <= 2147483647LL, "K=%d worst |C| %lld", K, worst);
+ CHECK(w::magmax(K) == 127 || (long long)K * 127 * 8 * (w::magmax(K) + 1) > 2147483647LL, "K=%d not tight", K);
+ }
+ CHECK(!w::fits(5120, 100) && !w::fits(100, 1024) && w::fits(256, 128), "fits");
+}
+
+static void test_bijection() {
+ const int K = 384, N = 256;
+ std::vector nib(w::w_bytes(K, N) * 2, 0), magc(w::w_bytes(K, N), 0), magcol(w::w_bytes(K, N), -1);
+ for (int k = 0; k < K; k++)
+ for (int n = 0; n < N; n++) {
+ const w::offsets o = w::index(K, N, k, n);
+ bool hi;
+ const size_t rb = restated_code_byte(K, k, n, &hi);
+ CHECK(o.code == rb && o.high == hi, "code (%d,%d): %zu/%d vs %zu/%d", k, n, o.code, o.high, rb, hi);
+ CHECK(o.mag == restated_mag_byte(K, k, n), "mag (%d,%d)", k, n);
+ CHECK(o.code % w::TILE < (size_t)w::CODE_BYTES && o.mag % w::TILE >= (size_t)w::CODE_BYTES, "regions");
+ nib[o.code * 2 + o.high]++;
+ magc[o.mag]++;
+ CHECK(magcol[o.mag] < 0 || magcol[o.mag] == n, "mag byte shared by two columns");
+ magcol[o.mag] = n;
+ }
+ size_t codes = 0, mags = 0;
+ for (size_t b = 0; b < w::w_bytes(K, N); b++) {
+ if (b % w::TILE < (size_t)w::CODE_BYTES) {
+ CHECK(nib[b * 2] == 1 && nib[b * 2 + 1] == 1, "byte %zu nibbles %d %d", b, nib[b * 2], nib[b * 2 + 1]);
+ codes++;
+ } else {
+ CHECK(magc[b] == 32, "mag byte %zu claimed %d times", b, magc[b]);
+ mags++;
+ }
+ }
+ CHECK(codes * 2 == (size_t)K * N && mags * 32 == (size_t)K * N, "coverage");
+}
+
+static void test_factor_pack() {
+ const int K = 256, N = 128;
+ std::mt19937 rng(7);
+ std::vector c(K * N);
+ std::vector s((K / 32) * N), B(K * N), S(K * N);
+ for (auto& x : c) x = (int8_t)((int)(rng() % 16) - 8);
+ for (auto& x : s) x = std::ldexp((float)((int)(rng() % 255) - 127), -14); // signed, exact in bf16
+ s[3] = 0.0f; // an all-zero group
+ for (int k = 0; k < K; k++)
+ for (int n = 0; n < N; n++) {
+ const float sg = s[(k / 32) * N + n];
+ S[k * N + n] = sg;
+ B[k * N + n] = sg == 0.0f ? 0.0f : (float)c[k * N + n] * sg;
+ }
+ std::vector c2(K * N);
+ std::vector s2((K / 32) * N);
+ CHECK(w::factor(B.data(), K, N, c2.data(), s2.data()), "factor refused a valid input");
+ // any exact form is acceptable: check B = c2 * s2 element for element, and |c2| fits int4
+ for (int k = 0; k < K; k++)
+ for (int n = 0; n < N; n++) {
+ const float sg = s2[(k / 32) * N + n];
+ CHECK(c2[k * N + n] >= -8 && c2[k * N + n] <= 7, "code range");
+ CHECK((float)c2[k * N + n] * sg == B[k * N + n], "factor (%d,%d) not exact", k, n);
+ }
+ // and where the group holds a -8 code the container's own (c, s) is the one found (q = 8 tried first)
+ int same = 0, groups = 0;
+ for (int g = 0; g < K / 32; g++)
+ for (int n = 0; n < N; n++) {
+ bool has8 = false;
+ for (int kk = 0; kk < 32; kk++) has8 |= c[(g * 32 + kk) * N + n] == -8;
+ if (!has8 || s[g * N + n] == 0.0f) continue;
+ ++groups;
+ same += s2[g * N + n] == s[g * N + n];
+ }
+ CHECK(groups > 0 && same == groups, "own scale recovered in %d of %d groups with a -8", same, groups);
+ // a group of only even codes may be represented at twice the scale: still exact, checked above
+ // refusals
+ auto refused = [&](std::vector b, const char* what) {
+ CHECK(!w::factor(b.data(), K, N, c2.data(), s2.data()), "factor accepted: %s", what);
+ };
+ { auto b = B; size_t i = 40 * N + 1; while (S[i] == 0.0f) i++; b[i] = 0.37f * S[i]; refused(b, "a non-multiple element"); }
+ { auto b = B; b[50 * N + 7] = std::nanf(""); refused(b, "NaN"); }
+ { auto b = B; for (int kk = 0; kk < 32; kk++) b[(64 + kk) * N + 3] = (float)(kk * 9 - 140); refused(b, "a 32-level ramp (needs > 4 bits)"); }
+
+ std::vector img(w::w_bytes(K, N));
+ std::vector cs(N);
+ CHECK(w::pack(c2.data(), s2.data(), K, N, img.data(), cs.data()), "pack");
+ const int mm = w::magmax(K);
+ for (int n = 0; n < N; n++) {
+ float amax = 0;
+ for (int g = 0; g < K / 32; g++) amax = std::fmax(amax, std::fabs(s2[g * N + n]));
+ CHECK(cs[n] == (amax > 0 ? amax / mm : 1.0f), "cs[%d]", n);
+ }
+ int maxmag = 0;
+ for (int k = 0; k < K; k++)
+ for (int n = 0; n < N; n++) {
+ int cc, mg;
+ w::read(img.data(), K, N, k, n, &cc, &mg);
+ CHECK(cc == c2[k * N + n], "read code (%d,%d)", k, n);
+ const long want = std::lrint(s2[(k / 32) * N + n] / cs[n]);
+ CHECK(mg == want, "mag (%d,%d): %d vs %ld", k, n, mg, want);
+ maxmag = std::max(maxmag, std::abs(mg));
+ }
+ CHECK(maxmag == mm, "the column max reaches MAGMAX (%d vs %d)", maxmag, mm);
+ std::vector bad = c2;
+ bad[17] = 8;
+ CHECK(!w::pack(bad.data(), s2.data(), K, N, img.data(), cs.data()), "pack accepted code 8");
+}
+
+static void test_a_and_ref() {
+ const int M = 64, K = 256, N = 128, ld = K + 32;
+ std::mt19937 rng(11);
+ std::vector a(M * ld), ap(M * K), c(K * N);
+ for (auto& x : a) x = (int8_t)((int)(rng() % 255) - 127);
+ for (auto& x : c) x = (int8_t)((int)(rng() % 16) - 8);
+ CHECK(w::pack_a(a.data(), ld, M, K, ap.data()), "pack_a");
+ for (int m = 0; m < M; m++)
+ for (int k = 0; k < K; k++) {
+ // A.reshape(M/32, 8, 4, K/128, 8, 16).transpose(0, 3, 1, 4, 2, 5)
+ const int rt = m / 32, mb = (m % 32) / 4, r = m % 4, ki = k / 128, kb = (k % 128) / 16, cc = k % 16;
+ const size_t o = (((((size_t)rt * (K / 128) + ki) * 8 + mb) * 8 + kb) * 4 + r) * 16 + cc;
+ CHECK(ap[o] == a[(size_t)m * ld + k], "pack_a (%d,%d)", m, k);
+ }
+ CHECK(!w::pack_a(a.data(), ld, 48, K, ap.data()), "pack_a accepted M=48");
+ std::vector s((K / 32) * N);
+ for (auto& x : s) x = (float)((int)(rng() % 255) - 127);
+ std::vector img(w::w_bytes(K, N));
+ std::vector cs(N);
+ CHECK(w::pack(c.data(), s.data(), K, N, img.data(), cs.data()), "pack");
+ for (int m = 0; m < M; m += 7)
+ for (int n = 0; n < N; n += 5) {
+ int64_t dense = 0;
+ for (int k = 0; k < K; k++) {
+ int cc, mg;
+ w::read(img.data(), K, N, k, n, &cc, &mg);
+ dense += (int64_t)a[(size_t)m * ld + k] * cc * mg;
+ }
+ CHECK(w::ref_dot(a.data() + (size_t)m * ld, img.data(), K, N, n) == dense, "ref_dot (%d,%d)", m, n);
+ }
+}
+
+static bool slurp(const std::string& p, std::vector& v) {
+ std::ifstream f(p, std::ios::binary);
+ if (!f) return false;
+ v.assign(std::istreambuf_iterator(f), {});
+ return true;
+}
+
+static void test_device_vectors() {
+ const char* e = std::getenv("W4G32_VEC");
+ if (!e || !*e) { std::printf(" (device-vector leg skipped: W4G32_VEC unset)\n"); return; }
+ char dir[4096];
+ int M, K, N;
+ if (std::sscanf(e, "%4095s %d %d %d", dir, &M, &K, &N) != 4) { CHECK(false, "W4G32_VEC=\"dir M K N\""); return; }
+ std::vector W, A, C;
+ if (!slurp(std::string(dir) + "/W.bin", W) || !slurp(std::string(dir) + "/A.bin", A) || !slurp(std::string(dir) + "/C_ref.bin", C)) {
+ CHECK(false, "cannot read %s/{W,A,C_ref}.bin", dir);
+ return;
+ }
+ CHECK(W.size() == w::w_bytes(K, N) && A.size() == (size_t)M * K && C.size() == (size_t)M * N * 4, "vector sizes");
+ if (fails) return;
+ // un-pack A with the restated order, check pack_a gives the file back, then sample C_ref
+ std::vector a((size_t)M * K), ap((size_t)M * K);
+ size_t o = 0;
+ for (int rt = 0; rt < M / 32; rt++)
+ for (int ki = 0; ki < K / 128; ki++)
+ for (int mb = 0; mb < 8; mb++)
+ for (int kb = 0; kb < 8; kb++)
+ for (int r = 0; r < 4; r++)
+ for (int cc = 0; cc < 16; cc++) a[(size_t)(rt * 32 + mb * 4 + r) * K + ki * 128 + kb * 16 + cc] = (int8_t)A[o++];
+ w::pack_a(a.data(), K, M, K, ap.data());
+ CHECK(std::memcmp(ap.data(), A.data(), A.size()) == 0, "pack_a does not reproduce the device's A.bin");
+ const int32_t* cr = reinterpret_cast(C.data());
+ int checked = 0;
+ for (int m = 0; m < M; m += 61)
+ for (int n = 0; n < N; n += 97) {
+ const int64_t v = w::ref_dot(a.data() + (size_t)m * K, reinterpret_cast(W.data()), K, N, n);
+ CHECK(v == cr[(size_t)m * N + n], "C_ref (%d,%d): %lld vs %d", m, n, (long long)v, cr[(size_t)m * N + n]);
+ ++checked;
+ }
+ std::printf(" device-vector leg: %s M=%d K=%d N=%d, %d C_ref entries reproduced\n", dir, M, K, N, checked);
+}
+
+int main() {
+ test_sizes();
+ test_bijection();
+ test_factor_pack();
+ test_a_and_ref();
+ test_device_vectors();
+ if (fails) { std::printf("npu_w4g32_layout_test: %d failure(s)\n", fails); return 1; }
+ std::printf("npu_w4g32_layout_test: OK\n");
+ return 0;
+}
diff --git a/tests/npu_w4g32_layout_test.sh b/tests/npu_w4g32_layout_test.sh
new file mode 100755
index 000000000..c4a4c1800
--- /dev/null
+++ b/tests/npu_w4g32_layout_test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# npu_w4g32_layout_test.sh
+#
+# The w4g32 weight image contract (q4nx_w4g32.h, the large-M int4 prefill tier) is pure index math: no
+# runtime, no device, no NPU toolchain. Same reason as npu_blocked_layout_test.sh:
+# the check compiles itself with the build's own compiler at test time, so it cannot
+# become "Not Run" by being absent from CI's explicit target list.
+set -u
+CXX=${1:?usage: npu_w4g32_layout_test.sh }
+INC=${2:?}
+SRC=${3:?}
+work=$(mktemp -d "${TMPDIR:-/tmp}/npu-w4g32-layout.XXXXXX")
+trap 'rm -rf "$work"' EXIT
+"$CXX" -std=c++17 -O1 -Wall -Wextra -I "$INC" "$SRC" -o "$work/t" || {
+ echo "npu_w4g32_layout_test: compile failed"; exit 1; }
+"$work/t"